428 files changed, 251277 insertions, 65753 deletions

diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index 2abb52194..f60fa85ce 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = cluster storage protocol performance debug features encryption mount
+SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system \
+          playground
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am
index c0b9141c6..90370d861 100644
--- a/xlators/bindings/python/src/Makefile.am
+++ b/xlators/bindings/python/src/Makefile.am
@@ -9,7 +9,7 @@ pythondir = $(xlatordir)/python
 
 python_so_SOURCES = python.c
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
+AM_CFLAGS = -fPIC $(GF_CPPFLAGS) -Wall \
 	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
 	$(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\"
 
diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py
index ee0eb1310..337c983ec 100644
--- a/xlators/bindings/python/src/gluster.py
+++ b/xlators/bindings/python/src/gluster.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
+
+#  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
 from ctypes import *
 from glustertypes import *
 from glusterstack import *
diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py
index ba24c8165..0c071ae98 100644
--- a/xlators/bindings/python/src/glusterstack.py
+++ b/xlators/bindings/python/src/glusterstack.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
+
+#  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
 from ctypes import *
 from glustertypes import *
 
diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py
index e9069d07c..98437d22e 100644
--- a/xlators/bindings/python/src/glustertypes.py
+++ b/xlators/bindings/python/src/glustertypes.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
+
+#  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+#  This file is part of GlusterFS.
 #
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+#  This file is licensed to you under your choice of the GNU Lesser
+#  General Public License, version 3 or any later version (LGPLv3 or
+#  later), or the GNU General Public License, version 2 (GPLv2), in all
+#  cases as published by the Free Software Foundation.
+
 from ctypes import *
 import collections
 
diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c
index 739ef7329..9b96790de 100644
--- a/xlators/bindings/python/src/python.c
+++ b/xlators/bindings/python/src/python.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include <Python.h>
 
 #ifndef _CONFIG_H
@@ -45,7 +35,7 @@ python_writev (call_frame_t *frame,
               xlator_t *this,
               fd_t *fd,
               struct iovec *vector,
-              int32_t count, 
+              int32_t count,
               off_t offset)
 {
   python_private_t *priv = (python_private_t *)this->private;
@@ -77,9 +67,6 @@ struct xlator_fops fops = {
     .writev       = python_writev
 };
 
-struct xlator_mops mops = {
-};
-
 static PyObject *
 AnonModule_FromFile (const char* fname)
 {
@@ -151,7 +138,7 @@ init (xlator_t *this)
   Py_InitializeEx(0);
 
   if (!this->children) {
-    gf_log ("python", GF_LOG_ERROR, 
+    gf_log ("python", GF_LOG_ERROR,
             "FATAL: python should have exactly one child");
     return -1;
   }
@@ -169,7 +156,7 @@ init (xlator_t *this)
   }
 
   priv->pInterp = Py_NewInterpreter();
-    
+
   // Adjust python's path
   PyObject *syspath = PySys_GetObject("path");
   PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH);
@@ -191,7 +178,7 @@ init (xlator_t *this)
   priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec");
 
   gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname);
-  
+
   priv->pScriptModule = AnonModule_FromFile(priv->scriptname);
   if (!priv->pScriptModule || PyErr_Occurred())
   {
@@ -220,7 +207,7 @@ init (xlator_t *this)
   return 0;
 }
 
-void 
+void
 fini (xlator_t *this)
 {
   python_private_t *priv = (python_private_t*)(this->private);
diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py
index 507455c85..59a991dca 100644
--- a/xlators/bindings/python/src/testxlator.py
+++ b/xlators/bindings/python/src/testxlator.py
@@ -1,19 +1,12 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
+"""
+  Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+"""
 
 """
 This is a test translator written in python.
diff --git a/xlators/cluster/Makefile.am b/xlators/cluster/Makefile.am
index a6ddb3564..0990822a7 100644
--- a/xlators/cluster/Makefile.am
+++ b/xlators/cluster/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = unify stripe afr dht ha map
+SUBDIRS = stripe afr dht
 
 CLEANFILES = 
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index 1bde9e5ba..35d18a6c0 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -1,20 +1,37 @@
-xlator_LTLIBRARIES = afr.la
+xlator_LTLIBRARIES = afr.la pump.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-afr_la_LDFLAGS = -module -avoidversion 
+afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
+	afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c \
+	afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c \
+	afr-self-heal-algorithm.c afr-lk-common.c afr-self-heald.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-afr_la_SOURCES = afr.c afr-dir-read.c afr-dir-write.c afr-inode-read.c afr-inode-write.c afr-transaction.c afr-self-heal-data.c afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c
+afr_la_LDFLAGS = -module -avoid-version
+afr_la_SOURCES = $(afr_common_source) afr.c
 afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h
+pump_la_LDFLAGS = -module -avoid-version
+pump_la_SOURCES =  $(afr_common_source) pump.c
+pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
+	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h \
+	afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c \
+	afr-self-heald.h $(top_builddir)/xlators/lib/src/libxlator.h \
+	$(top_builddir)/glusterfsd/src/glusterfsd.h
 
-CLEANFILES = 
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
 
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/replicate.so
+	rm -f $(DESTDIR)$(xlatordir)/pump.so
 
 install-data-hook:
-	ln -sf afr.so $(DESTDIR)$(xlatordir)/replicate.so
\ No newline at end of file
+	ln -sf afr.so $(DESTDIR)$(xlatordir)/replicate.so
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
new file mode 100644
index 000000000..af01f2ef2
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -0,0 +1,4591 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <libgen.h>
+#include <unistd.h>
+#include <fnmatch.h>
+#include <sys/time.h>
+#include <stdlib.h>
+#include <signal.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "afr.h"
+#include "dict.h"
+#include "xlator.h"
+#include "hashfn.h"
+#include "logging.h"
+#include "stack.h"
+#include "list.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "common-utils.h"
+#include "compat-errno.h"
+#include "compat.h"
+#include "byte-order.h"
+#include "statedump.h"
+#include "inode.h"
+
+#include "fd.h"
+
+#include "afr-inode-read.h"
+#include "afr-inode-write.h"
+#include "afr-dir-read.h"
+#include "afr-dir-write.h"
+#include "afr-transaction.h"
+#include "afr-self-heal.h"
+#include "afr-self-heal-common.h"
+#include "afr-self-heald.h"
+#include "pump.h"
+
+#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000100000000ULL
+#define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
+#define AFR_STATISTICS_HISTORY_SIZE    50
+int
+afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
+                                gf_boolean_t fail_conflict);
+void
+afr_children_copy (int32_t *dst, int32_t *src, unsigned int child_count)
+{
+        int     i = 0;
+
+        for (i = 0; i < child_count; i++)
+                dst[i] = src[i];
+}
+
+void
+afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path)
+{
+        int             i           = 0;
+        afr_private_t   *priv       = NULL;
+        int             ret         = 0;
+
+        priv   = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
+                                       3 * sizeof(int32_t));
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: Unable to set dict value for %s",
+                                path, priv->pending_key[i]);
+                /* 3 = data+metadata+entry */
+        }
+        ret = dict_set_int32 (xattr_req, GF_GFIDLESS_LOOKUP, 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s: failed to set gfidless "
+                        "lookup", path);
+        }
+}
+
+int
+afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
+                              dict_t *xattr_req, loc_t *loc, void **gfid_req)
+{
+        int     ret = -ENOMEM;
+
+        GF_ASSERT (gfid_req);
+
+        *gfid_req = NULL;
+        local->xattr_req = dict_new ();
+        if (!local->xattr_req)
+                goto out;
+        if (xattr_req)
+                dict_copy (xattr_req, local->xattr_req);
+
+        afr_xattr_req_prepare (this, local->xattr_req, loc->path);
+        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_INODELK_COUNT);
+        }
+        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
+        }
+
+        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
+        }
+
+        ret = dict_get_ptr (local->xattr_req, "gfid-req", gfid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: failed to get the gfid from dict", loc->path);
+                *gfid_req = NULL;
+        } else {
+                if (loc->parent != NULL)
+                        dict_del (local->xattr_req, "gfid-req");
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+afr_lookup_save_gfid (uuid_t dst, void* new, const loc_t *loc)
+{
+        inode_t  *inode = NULL;
+
+        inode = loc->inode;
+        if (inode && !uuid_is_null (inode->gfid))
+                uuid_copy (dst, inode->gfid);
+        else if (!uuid_is_null (loc->gfid))
+                uuid_copy (dst, loc->gfid);
+        else if (new && !uuid_is_null (new))
+                uuid_copy (dst, new);
+}
+
+int
+afr_errno_count (int32_t *children, int *child_errno,
+                 unsigned int child_count, int32_t op_errno)
+{
+        int i = 0;
+        int errno_count = 0;
+        int child = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (children) {
+                        child = children[i];
+                        if (child == -1)
+                                break;
+                } else {
+                        child = i;
+                }
+                if (child_errno[child] == op_errno)
+                        errno_count++;
+        }
+        return errno_count;
+}
+
+int32_t
+afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
+{
+        int ret       = 0;
+        uuid_t *pgfid = NULL;
+
+        GF_ASSERT (gfid);
+
+        pgfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
+        if (!pgfid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*pgfid, gfid);
+
+        ret = dict_set_dynptr (dict, "gfid-req", pgfid, sizeof (uuid_t));
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "gfid set failed");
+
+out:
+        if (ret && pgfid)
+                GF_FREE (pgfid);
+
+        return ret;
+}
+
+void
+afr_inode_ctx_destroy (afr_inode_ctx_t *ctx)
+{
+        if (!ctx)
+                return;
+        GF_FREE (ctx->fresh_children);
+        GF_FREE (ctx);
+}
+
+afr_inode_ctx_t*
+__afr_inode_ctx_get (inode_t *inode, xlator_t *this)
+{
+        int             ret      = 0;
+        uint64_t        ctx_addr = 0;
+        afr_inode_ctx_t *ctx     = NULL;
+        afr_private_t   *priv    = NULL;
+
+        priv = this->private;
+        ret = __inode_ctx_get (inode, this, &ctx_addr);
+        if (ret < 0)
+                ctx_addr = 0;
+        if (ctx_addr != 0) {
+                ctx = (afr_inode_ctx_t*) (long) ctx_addr;
+                goto out;
+        }
+        ctx = GF_CALLOC (1, sizeof (*ctx),
+                         gf_afr_mt_inode_ctx_t);
+        if (!ctx)
+                goto fail;
+        ctx->fresh_children = GF_CALLOC (priv->child_count,
+                                         sizeof (*ctx->fresh_children),
+                                         gf_afr_mt_int32_t);
+        if (!ctx->fresh_children)
+                goto fail;
+        ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
+        if (ret) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
+                                  "set the inode ctx (%s)",
+                                  uuid_utoa (inode->gfid));
+                goto fail;
+        }
+
+out:
+        return ctx;
+
+fail:
+        afr_inode_ctx_destroy (ctx);
+        return NULL;
+}
+
+afr_inode_ctx_t*
+afr_inode_ctx_get (inode_t *inode, xlator_t *this)
+{
+        afr_inode_ctx_t *ctx = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __afr_inode_ctx_get (inode, this);
+        }
+        UNLOCK (&inode->lock);
+        return ctx;
+}
+
+void
+afr_inode_get_ctx_params (xlator_t *this, inode_t *inode,
+                          afr_inode_params_t *params)
+{
+        GF_ASSERT (inode);
+        GF_ASSERT (params);
+
+        afr_inode_ctx_t *ctx = NULL;
+        afr_private_t   *priv = NULL;
+        int             i = 0;
+        int32_t         read_child = -1;
+        int32_t         *fresh_children = NULL;
+
+        priv = this->private;
+        LOCK (&inode->lock);
+        {
+                ctx = __afr_inode_ctx_get (inode, this);
+                if (!ctx)
+                        goto unlock;
+                switch (params->op) {
+                case AFR_INODE_GET_READ_CTX:
+                        fresh_children = params->u.read_ctx.children;
+                        read_child = (int32_t)(ctx->masks &
+                                               AFR_ICTX_READ_CHILD_MASK);
+                        params->u.read_ctx.read_child = read_child;
+                        if (!fresh_children)
+                                goto unlock;
+                        for (i = 0; i < priv->child_count; i++)
+                                fresh_children[i] = ctx->fresh_children[i];
+                        break;
+                case AFR_INODE_GET_OPENDIR_DONE:
+                        params->u.value = _gf_false;
+                        if (ctx->masks & AFR_ICTX_OPENDIR_DONE_MASK)
+                                params->u.value = _gf_true;
+                        break;
+                default:
+                        GF_ASSERT (0);
+                        break;
+                }
+        }
+unlock:
+        UNLOCK (&inode->lock);
+}
+
+gf_boolean_t
+afr_is_split_brain (xlator_t *this, inode_t *inode)
+{
+        afr_inode_ctx_t *ctx = NULL;
+        gf_boolean_t    spb  = _gf_false;
+
+        ctx = afr_inode_ctx_get (inode, this);
+        if (!ctx)
+                goto out;
+        if ((ctx->mdata_spb == SPB) || (ctx->data_spb == SPB))
+                spb = _gf_true;
+out:
+        return spb;
+}
+
+gf_boolean_t
+afr_is_opendir_done (xlator_t *this, inode_t *inode)
+{
+        afr_inode_params_t params = {0};
+
+        params.op = AFR_INODE_GET_OPENDIR_DONE;
+        afr_inode_get_ctx_params (this, inode, &params);
+        return params.u.value;
+}
+
+int32_t
+afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children)
+{
+        afr_inode_params_t      params = {0};
+
+        params.op = AFR_INODE_GET_READ_CTX;
+        params.u.read_ctx.children = fresh_children;
+        afr_inode_get_ctx_params (this, inode, &params);
+        return params.u.read_ctx.read_child;
+}
+
+void
+afr_inode_ctx_set_read_child (afr_inode_ctx_t *ctx, int32_t read_child)
+{
+        uint64_t        remaining_mask = 0;
+        uint64_t        mask         = 0;
+
+        remaining_mask = (~AFR_ICTX_READ_CHILD_MASK & ctx->masks);
+        mask = (AFR_ICTX_READ_CHILD_MASK & read_child);
+        ctx->masks = remaining_mask | mask;
+}
+
+void
+afr_inode_ctx_set_read_ctx (afr_inode_ctx_t *ctx, int32_t read_child,
+                            int32_t *fresh_children, int32_t child_count)
+{
+        int             i            = 0;
+
+        afr_inode_ctx_set_read_child (ctx, read_child);
+        for (i = 0; i < child_count; i++) {
+                if (fresh_children)
+                        ctx->fresh_children[i] = fresh_children[i];
+                else
+                        ctx->fresh_children[i] = -1;
+        }
+}
+
+void
+afr_inode_ctx_rm_stale_children (afr_inode_ctx_t *ctx, int32_t *stale_children,
+                                 int32_t child_count)
+{
+        int             i            = 0;
+        int32_t         read_child   = -1;
+
+        GF_ASSERT (stale_children);
+        for (i = 0; i < child_count; i++) {
+                if (stale_children[i] == -1)
+                        break;
+                afr_children_rm_child (ctx->fresh_children,
+                                       stale_children[i], child_count);
+        }
+        read_child = (int32_t)(ctx->masks & AFR_ICTX_READ_CHILD_MASK);
+        if (!afr_is_child_present (ctx->fresh_children, child_count,
+                                   read_child))
+                afr_inode_ctx_set_read_child (ctx, ctx->fresh_children[0]);
+}
+
+void
+afr_inode_ctx_set_opendir_done (afr_inode_ctx_t *ctx)
+{
+        uint64_t        remaining_mask = 0;
+        uint64_t        mask = 0;
+
+        remaining_mask = (~AFR_ICTX_OPENDIR_DONE_MASK & ctx->masks);
+        mask = (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_OPENDIR_DONE_MASK);
+        ctx->masks = remaining_mask | mask;
+}
+
+void
+afr_inode_set_ctx_params (xlator_t *this, inode_t *inode,
+                          afr_inode_params_t *params)
+{
+        GF_ASSERT (inode);
+        GF_ASSERT (params);
+
+        afr_inode_ctx_t *ctx            = NULL;
+        afr_private_t   *priv           = NULL;
+        int32_t         read_child      = -1;
+        int32_t         *fresh_children = NULL;
+        int32_t         *stale_children = NULL;
+
+        priv = this->private;
+        LOCK (&inode->lock);
+        {
+                ctx = __afr_inode_ctx_get (inode, this);
+                if (!ctx)
+                        goto unlock;
+                switch (params->op) {
+                case AFR_INODE_SET_READ_CTX:
+                        read_child = params->u.read_ctx.read_child;
+                        fresh_children = params->u.read_ctx.children;
+                        afr_inode_ctx_set_read_ctx (ctx, read_child,
+                                                    fresh_children,
+                                                    priv->child_count);
+                        break;
+                case AFR_INODE_RM_STALE_CHILDREN:
+                        stale_children = params->u.read_ctx.children;
+                        afr_inode_ctx_rm_stale_children (ctx,
+                                                         stale_children,
+                                                         priv->child_count);
+                        break;
+                case AFR_INODE_SET_OPENDIR_DONE:
+                        afr_inode_ctx_set_opendir_done (ctx);
+                        break;
+                default:
+                        GF_ASSERT (0);
+                        break;
+                }
+        }
+unlock:
+        UNLOCK (&inode->lock);
+}
+
+void
+afr_set_split_brain (xlator_t *this, inode_t *inode, afr_spb_state_t mdata_spb,
+                     afr_spb_state_t data_spb)
+{
+        afr_inode_ctx_t *ctx = NULL;
+
+        ctx = afr_inode_ctx_get (inode, this);
+        if (mdata_spb != DONT_KNOW)
+                ctx->mdata_spb = mdata_spb;
+        if (data_spb != DONT_KNOW)
+                ctx->data_spb = data_spb;
+}
+
+void
+afr_set_opendir_done (xlator_t *this, inode_t *inode)
+{
+        afr_inode_params_t params = {0};
+
+        params.op = AFR_INODE_SET_OPENDIR_DONE;
+        afr_inode_set_ctx_params (this, inode, &params);
+}
+
+void
+afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
+                        int32_t *fresh_children)
+{
+        afr_inode_params_t params = {0};
+        afr_private_t      *priv  = NULL;
+
+        priv = this->private;
+        GF_ASSERT (read_child >= 0);
+        GF_ASSERT (fresh_children);
+        GF_ASSERT (afr_is_child_present (fresh_children, priv->child_count,
+                                         read_child));
+
+        params.op = AFR_INODE_SET_READ_CTX;
+        params.u.read_ctx.read_child     = read_child;
+        params.u.read_ctx.children = fresh_children;
+        afr_inode_set_ctx_params (this, inode, &params);
+}
+
+void
+afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
+                             int32_t *stale_children)
+{
+        afr_inode_params_t params = {0};
+
+        GF_ASSERT (stale_children);
+
+        params.op = AFR_INODE_RM_STALE_CHILDREN;
+        params.u.read_ctx.children = stale_children;
+        afr_inode_set_ctx_params (this, inode, &params);
+}
+
+gf_boolean_t
+afr_is_source_child (int32_t *sources, int32_t child_count, int32_t child)
+{
+        gf_boolean_t             source_xattrs = _gf_false;
+
+        GF_ASSERT (child < child_count);
+
+        if ((child >= 0) && (child < child_count) &&
+             sources[child]) {
+                source_xattrs = _gf_true;
+        }
+        return source_xattrs;
+}
+
+gf_boolean_t
+afr_is_child_present (int32_t *success_children, int32_t child_count,
+                      int32_t child)
+{
+        gf_boolean_t             success_child = _gf_false;
+        int                      i = 0;
+
+        GF_ASSERT (child < child_count);
+
+        for (i = 0; i < child_count; i++) {
+                if (success_children[i] == -1)
+                        break;
+                if (child == success_children[i]) {
+                        success_child = _gf_true;
+                        break;
+                }
+        }
+        return success_child;
+}
+
+gf_boolean_t
+afr_is_read_child (int32_t *success_children, int32_t *sources,
+                   int32_t child_count, int32_t child)
+{
+        gf_boolean_t             success_child = _gf_false;
+        gf_boolean_t             source        = _gf_false;
+
+        if (child < 0) {
+                return _gf_false;
+        }
+
+        GF_ASSERT (success_children);
+        GF_ASSERT (child_count > 0);
+
+        success_child = afr_is_child_present (success_children, child_count,
+                                              child);
+        if (!success_child)
+                goto out;
+        if (NULL == sources) {
+                source = _gf_true;
+                goto out;
+        }
+        source = afr_is_source_child (sources, child_count, child);
+out:
+        return (success_child && source);
+}
+
+int32_t
+afr_hash_child (int32_t *success_children, int32_t child_count,
+                unsigned int hmode, uuid_t gfid)
+{
+        uuid_t  gfid_copy = {0,};
+        pid_t pid;
+
+        if (!hmode) {
+                return -1;
+        }
+
+        if (gfid) {
+               uuid_copy(gfid_copy,gfid);
+        }
+        if (hmode > 1) {
+                /*
+                 * Why getpid?  Because it's one of the cheapest calls
+                 * available - faster than gethostname etc. - and returns a
+                 * constant-length value that's sure to be shorter than a UUID.
+                 * It's still very unlikely to be the same across clients, so
+                 * it still provides good mixing.  We're not trying for
+                 * perfection here.  All we need is a low probability that
+                 * multiple clients won't converge on the same subvolume.
+                 */
+                pid = getpid();
+                memcpy (gfid_copy, &pid, sizeof(pid));
+        }
+
+        return SuperFastHash((char *)gfid_copy,
+                             sizeof(gfid_copy)) % child_count;
+}
+
+/* If sources is NULL the xattrs are assumed to be of source for all
+ * success_children.
+ */
+int
+afr_select_read_child_from_policy (int32_t *success_children,
+                                   int32_t child_count, int32_t prev_read_child,
+                                   int32_t config_read_child, int32_t *sources,
+                                   unsigned int hmode, uuid_t gfid)
+{
+        int32_t                  read_child   = -1;
+        int                      i            = 0;
+
+        GF_ASSERT (success_children);
+
+        read_child = config_read_child;
+        if (afr_is_read_child (success_children, sources, child_count,
+                               read_child))
+                goto out;
+
+        read_child = prev_read_child;
+        if (afr_is_read_child (success_children, sources, child_count,
+                               read_child))
+                goto out;
+
+        read_child = afr_hash_child (success_children, child_count,
+                                     hmode, gfid);
+        if (afr_is_read_child (success_children, sources, child_count,
+                               read_child)) {
+                goto out;
+        }
+
+        for (i = 0; i < child_count; i++) {
+                read_child = success_children[i];
+                if (read_child < 0)
+                        break;
+                if (afr_is_read_child (success_children, sources, child_count,
+                                       read_child))
+                        goto out;
+        }
+        read_child = -1;
+
+out:
+        return read_child;
+}
+
+/* This function should be used when all the success_children are sources
+ */
+void
+afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
+                              int32_t *fresh_children, int32_t prev_read_child,
+                              int32_t config_read_child, uuid_t gfid)
+{
+        int                      read_child = -1;
+        afr_private_t            *priv = NULL;
+
+        priv = this->private;
+        read_child = afr_select_read_child_from_policy (fresh_children,
+                                                        priv->child_count,
+                                                        prev_read_child,
+                                                        config_read_child,
+                                                        NULL,
+                                                        priv->hash_mode, gfid);
+        if (read_child >= 0)
+                afr_inode_set_read_ctx (this, inode, read_child,
+                                        fresh_children);
+}
+
+/* afr_next_call_child ()
+ * This is a common function used by all the read-type fops
+ * This function should not be called with the inode's read_children array.
+ * The fop's handler should make a copy of the inode's read_children,
+ * preferred read_child into the local vars, because while this function is
+ * in execution there is a chance for inode's read_ctx to change.
+ */
+int32_t
+afr_next_call_child (int32_t *fresh_children, unsigned char *child_up,
+                     size_t child_count, int32_t *last_index,
+                     int32_t read_child)
+{
+        int             next_index      = 0;
+        int32_t         next_call_child = -1;
+
+        GF_ASSERT (last_index);
+
+        next_index = *last_index;
+retry:
+        next_index++;
+        if ((next_index >= child_count) ||
+           (fresh_children[next_index] == -1))
+                goto out;
+        if ((fresh_children[next_index] == read_child) ||
+           (!child_up[fresh_children[next_index]]))
+                goto retry;
+        *last_index = next_index;
+        next_call_child = fresh_children[next_index];
+out:
+        return next_call_child;
+}
+
+ /* This function should not be called with the inode's read_children array.
+ * The fop's handler should make a copy of the inode's read_children,
+ * preferred read_child into the local vars, because while this function is
+ * in execution there is a chance for inode's read_ctx to change.
+ */
+int32_t
+afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
+                    int32_t *fresh_children,
+                    int32_t *call_child, int32_t *last_index)
+{
+        int             ret   = 0;
+        afr_private_t   *priv = NULL;
+        int             i     = 0;
+
+        GF_ASSERT (child_up);
+        GF_ASSERT (call_child);
+        GF_ASSERT (last_index);
+        GF_ASSERT (fresh_children);
+
+        if (read_child < 0) {
+                ret = -EIO;
+                goto out;
+        }
+        priv = this->private;
+        *call_child = -1;
+        *last_index = -1;
+
+        if (child_up[read_child]) {
+                *call_child = read_child;
+        } else {
+                for (i = 0; i < priv->child_count; i++) {
+                        if (fresh_children[i] == -1)
+                                break;
+                        if (child_up[fresh_children[i]]) {
+                                *call_child = fresh_children[i];
+                                ret = 0;
+                                break;
+                        }
+                }
+
+                if (*call_child == -1) {
+                        ret = -ENOTCONN;
+                        goto out;
+                }
+
+                *last_index = i;
+        }
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d, call_child: %d, "
+                "last_index: %d", ret, *call_child, *last_index);
+        return ret;
+}
+
+void
+afr_reset_xattr (dict_t **xattr, unsigned int child_count)
+{
+        unsigned int i = 0;
+
+        if (!xattr)
+                goto out;
+        for (i = 0; i < child_count; i++) {
+                if (xattr[i]) {
+                        dict_unref (xattr[i]);
+                        xattr[i] = NULL;
+                }
+        }
+out:
+        return;
+}
+
+void
+afr_xattr_array_destroy (dict_t **xattr, unsigned int child_count)
+{
+        afr_reset_xattr (xattr, child_count);
+        GF_FREE (xattr);
+}
+
+void
+afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
+{
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+
+        sh = &local->self_heal;
+        priv = this->private;
+
+        if (sh->data_sh_info && strcmp (sh->data_sh_info, ""))
+                GF_FREE (sh->data_sh_info);
+
+        if (sh->metadata_sh_info && strcmp (sh->metadata_sh_info, ""))
+                GF_FREE (sh->metadata_sh_info);
+
+        GF_FREE (sh->buf);
+
+        GF_FREE (sh->parentbufs);
+
+        if (sh->inode)
+                inode_unref (sh->inode);
+
+        afr_xattr_array_destroy (sh->xattr, priv->child_count);
+
+        GF_FREE (sh->child_errno);
+
+        afr_matrix_cleanup (sh->pending_matrix, priv->child_count);
+        afr_matrix_cleanup (sh->delta_matrix, priv->child_count);
+
+        GF_FREE (sh->sources);
+
+        GF_FREE (sh->success);
+
+        GF_FREE (sh->locked_nodes);
+
+        if (sh->healing_fd) {
+                fd_unref (sh->healing_fd);
+                sh->healing_fd = NULL;
+        }
+
+        GF_FREE ((char *)sh->linkname);
+
+        GF_FREE (sh->success_children);
+
+        GF_FREE (sh->fresh_children);
+
+        GF_FREE (sh->fresh_parent_dirs);
+
+        loc_wipe (&sh->parent_loc);
+        loc_wipe (&sh->lookup_loc);
+
+        GF_FREE (sh->checksum);
+
+        GF_FREE (sh->write_needed);
+        if (sh->healing_fd)
+                fd_unref (sh->healing_fd);
+}
+
+
+void
+afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
+
+        priv = this->private;
+
+        afr_matrix_cleanup (local->pending, priv->child_count);
+        afr_matrix_cleanup (local->transaction.txn_changelog,
+                            priv->child_count);
+
+        GF_FREE (local->internal_lock.locked_nodes);
+
+        for (i = 0; local->internal_lock.inodelk[i].domain; i++) {
+                GF_FREE (local->internal_lock.inodelk[i].locked_nodes);
+        }
+
+        GF_FREE (local->internal_lock.lower_locked_nodes);
+
+        afr_entry_lockee_cleanup (&local->internal_lock);
+
+        GF_FREE (local->transaction.pre_op);
+        GF_FREE (local->transaction.eager_lock);
+
+        GF_FREE (local->transaction.basename);
+        GF_FREE (local->transaction.new_basename);
+
+        loc_wipe (&local->transaction.parent_loc);
+        loc_wipe (&local->transaction.new_parent_loc);
+
+        GF_FREE (local->transaction.postop_piggybacked);
+}
+
+
+void
+afr_local_cleanup (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t * priv = NULL;
+
+        if (!local)
+                return;
+
+        afr_local_sh_cleanup (local, this);
+
+        afr_local_transaction_cleanup (local, this);
+
+        priv = this->private;
+
+        loc_wipe (&local->loc);
+        loc_wipe (&local->newloc);
+
+        if (local->fd)
+                fd_unref (local->fd);
+
+        if (local->xattr_req)
+                dict_unref (local->xattr_req);
+
+        if (local->dict)
+                dict_unref (local->dict);
+
+	GF_FREE(local->replies);
+
+        GF_FREE (local->child_up);
+
+        GF_FREE (local->child_errno);
+
+        GF_FREE (local->fresh_children);
+
+        { /* lookup */
+                if (local->cont.lookup.xattrs) {
+                        afr_reset_xattr (local->cont.lookup.xattrs,
+                                         priv->child_count);
+                        GF_FREE (local->cont.lookup.xattrs);
+                        local->cont.lookup.xattrs = NULL;
+                }
+
+                if (local->cont.lookup.xattr) {
+                        dict_unref (local->cont.lookup.xattr);
+                }
+
+                if (local->cont.lookup.inode) {
+                        inode_unref (local->cont.lookup.inode);
+                }
+
+                GF_FREE (local->cont.lookup.postparents);
+
+                GF_FREE (local->cont.lookup.bufs);
+
+                GF_FREE (local->cont.lookup.success_children);
+
+                GF_FREE (local->cont.lookup.sources);
+                afr_matrix_cleanup (local->cont.lookup.pending_matrix,
+                                    priv->child_count);
+        }
+
+        { /* getxattr */
+                GF_FREE (local->cont.getxattr.name);
+        }
+
+        { /* lk */
+                GF_FREE (local->cont.lk.locked_nodes);
+        }
+
+        { /* create */
+                if (local->cont.create.fd)
+                        fd_unref (local->cont.create.fd);
+                if (local->cont.create.params)
+                        dict_unref (local->cont.create.params);
+        }
+
+        { /* mknod */
+                if (local->cont.mknod.params)
+                        dict_unref (local->cont.mknod.params);
+        }
+
+        { /* mkdir */
+                if (local->cont.mkdir.params)
+                        dict_unref (local->cont.mkdir.params);
+        }
+
+        { /* symlink */
+                if (local->cont.symlink.params)
+                        dict_unref (local->cont.symlink.params);
+        }
+
+        { /* writev */
+                GF_FREE (local->cont.writev.vector);
+        }
+
+        { /* setxattr */
+                if (local->cont.setxattr.dict)
+                        dict_unref (local->cont.setxattr.dict);
+        }
+
+        { /* fsetxattr */
+                if (local->cont.fsetxattr.dict)
+                        dict_unref (local->cont.fsetxattr.dict);
+        }
+
+        { /* removexattr */
+                GF_FREE (local->cont.removexattr.name);
+        }
+        { /* xattrop */
+                if (local->cont.xattrop.xattr)
+                        dict_unref (local->cont.xattrop.xattr);
+        }
+        { /* fxattrop */
+                if (local->cont.fxattrop.xattr)
+                        dict_unref (local->cont.fxattrop.xattr);
+        }
+        { /* symlink */
+                GF_FREE (local->cont.symlink.linkpath);
+        }
+
+        { /* opendir */
+                GF_FREE (local->cont.opendir.checksum);
+        }
+
+        { /* readdirp */
+                if (local->cont.readdir.dict)
+                        dict_unref (local->cont.readdir.dict);
+        }
+
+        if (local->xdata_req)
+                dict_unref (local->xdata_req);
+
+        if (local->xdata_rsp)
+                dict_unref (local->xdata_rsp);
+}
+
+
+int
+afr_frame_return (call_frame_t *frame)
+{
+        afr_local_t *local = NULL;
+        int          call_count = 0;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                call_count = --local->call_count;
+        }
+        UNLOCK (&frame->lock);
+
+        return call_count;
+}
+
+int
+afr_set_elem_count_get (unsigned char *elems, int child_count)
+{
+        int i   = 0;
+        int ret = 0;
+
+        for (i = 0; i < child_count; i++)
+                if (elems[i])
+                        ret++;
+        return ret;
+}
+
+/**
+ * up_children_count - return the number of children that are up
+ */
+
+unsigned int
+afr_up_children_count (unsigned char *child_up, unsigned int child_count)
+{
+        return afr_set_elem_count_get (child_up, child_count);
+}
+
+unsigned int
+afr_locked_children_count (unsigned char *children, unsigned int child_count)
+{
+        return afr_set_elem_count_get (children, child_count);
+}
+
+unsigned int
+afr_pre_op_done_children_count (unsigned char *pre_op,
+                                unsigned int child_count)
+{
+        return afr_set_elem_count_get (pre_op, child_count);
+}
+
+gf_boolean_t
+afr_is_fresh_lookup (loc_t *loc, xlator_t *this)
+{
+        uint64_t          ctx = 0;
+        int32_t           ret = 0;
+
+        GF_ASSERT (loc);
+        GF_ASSERT (this);
+        GF_ASSERT (loc->inode);
+
+        ret = inode_ctx_get (loc->inode, this, &ctx);
+        if (0 == ret)
+                return _gf_false;
+        return _gf_true;
+}
+
+void
+afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
+{
+        GF_ASSERT (loc);
+        GF_ASSERT (buf);
+
+        uuid_copy (loc->gfid, buf->ia_gfid);
+        if (postparent)
+                uuid_copy (loc->pargfid, postparent->ia_gfid);
+}
+
+/*
+ * Quota size xattrs are not maintained by afr. There is a
+ * possibility that they differ even when both the directory changelog xattrs
+ * suggest everything is fine. So if there is at least one 'source' check among
+ * the sources which has the maximum quota size. Otherwise check among all the
+ * available ones for maximum quota size. This way if there is a source and
+ * stale copies it always votes for the 'source'.
+ * */
+
+static void
+afr_handle_quota_size (afr_local_t *local, xlator_t *this,
+                       dict_t *rsp_dict)
+{
+        int32_t       *sources       = NULL;
+        dict_t        *xattr         = NULL;
+        data_t        *max_data      = NULL;
+        int64_t       max_quota_size = -1;
+        data_t        *data          = NULL;
+        int64_t       *size          = NULL;
+        int64_t       quota_size     = -1;
+        afr_private_t *priv          = NULL;
+        int           i              = 0;
+        int           ret            = -1;
+        gf_boolean_t  source_present = _gf_false;
+
+        priv    = this->private;
+        sources = local->cont.lookup.sources;
+
+        if (rsp_dict == NULL) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "%s: Invalid "
+                                  "response dictionary", local->loc.path);
+                return;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sources[i]) {
+                        source_present = _gf_true;
+                        break;
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                /*
+                 * If there is at least one source lets check
+                 * for maximum quota sizes among sources, otherwise take the
+                 * maximum of the ones present to be on the safer side.
+                 */
+                if (source_present && !sources[i])
+                        continue;
+
+                xattr = local->cont.lookup.xattrs[i];
+                if (!xattr)
+                        continue;
+
+                data = dict_get (xattr, QUOTA_SIZE_KEY);
+                if (!data)
+                        continue;
+
+                size = (int64_t*)data->data;
+                quota_size = ntoh64(*size);
+                gf_log (this->name, GF_LOG_DEBUG, "%s: %d, size: %"PRId64,
+                        local->loc.path, i, quota_size);
+                if (quota_size > max_quota_size) {
+                        if (max_data)
+                                data_unref (max_data);
+
+                        max_quota_size = quota_size;
+                        max_data = data_ref (data);
+                }
+        }
+
+        if (max_data) {
+                ret = dict_set (rsp_dict, QUOTA_SIZE_KEY, max_data);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                                "quota size", local->loc.path);
+                }
+
+                data_unref (max_data);
+        }
+}
+
+int
+afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
+{
+        struct iatt     *buf = NULL;
+        struct iatt     *postparent = NULL;
+        dict_t          **xattr = NULL;
+        int32_t         *success_children = NULL;
+        int32_t         *sources = NULL;
+        afr_private_t   *priv = NULL;
+        int32_t         read_child = -1;
+        int             ret = 0;
+        int             i = 0;
+
+        GF_ASSERT (local);
+
+        buf = &local->cont.lookup.buf;
+        postparent = &local->cont.lookup.postparent;
+        xattr = &local->cont.lookup.xattr;
+        priv = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->cont.lookup.inode,
+                                             local->fresh_children);
+        if (read_child < 0) {
+                ret = -1;
+                goto out;
+        }
+        success_children = local->cont.lookup.success_children;
+        sources = local->cont.lookup.sources;
+        memset (sources, 0, sizeof (*sources) * priv->child_count);
+        afr_children_intersection_get (local->fresh_children, success_children,
+                                       sources, priv->child_count);
+        if (!sources[read_child]) {
+                read_child = -1;
+                for (i = 0; i < priv->child_count; i++) {
+                        if (sources[i]) {
+                                read_child = i;
+                                break;
+                        }
+                }
+        }
+        if (read_child < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Building lookup response from %d",
+                read_child);
+        if (!*xattr)
+                *xattr = dict_ref (local->cont.lookup.xattrs[read_child]);
+
+        *buf = local->cont.lookup.bufs[read_child];
+        *postparent = local->cont.lookup.postparents[read_child];
+
+        if (dict_get (local->xattr_req, QUOTA_SIZE_KEY))
+                afr_handle_quota_size (local, this, *xattr);
+
+        if (IA_INVAL == local->cont.lookup.inode->ia_type) {
+                /* fix for RT #602 */
+                local->cont.lookup.inode->ia_type = buf->ia_type;
+        }
+out:
+        return ret;
+}
+
+static void
+afr_lookup_update_lk_counts (afr_local_t *local, xlator_t *this,
+                            int child_index, dict_t *xattr)
+{
+        uint32_t inodelk_count = 0;
+        uint32_t entrylk_count = 0;
+        int      ret           = -1;
+        uint32_t parent_entrylk = 0;
+
+        GF_ASSERT (local);
+        GF_ASSERT (this);
+        GF_ASSERT (xattr);
+        GF_ASSERT (child_index >= 0);
+
+        ret = dict_get_uint32 (xattr, GLUSTERFS_INODELK_COUNT,
+                               &inodelk_count);
+        if (ret == 0)
+                local->inodelk_count += inodelk_count;
+
+        ret = dict_get_uint32 (xattr, GLUSTERFS_ENTRYLK_COUNT,
+                               &entrylk_count);
+        if (ret == 0)
+                local->entrylk_count += entrylk_count;
+        ret = dict_get_uint32 (xattr, GLUSTERFS_PARENT_ENTRYLK,
+                               &parent_entrylk);
+        if (!ret)
+                local->cont.lookup.parent_entrylk += parent_entrylk;
+}
+
+/*
+ * It's important to maintain a commutative property on do_*_self_heal and
+ * found*; once set, they must not be cleared by a subsequent iteration or
+ * call, so that they represent a logical OR of all iterations and calls
+ * regardless of child/key order.  That allows the caller to call us multiple
+ * times without having to use a separate variable as a "reduce" accumulator.
+ */
+static void
+afr_lookup_set_self_heal_params_by_xattr (afr_local_t *local, xlator_t *this,
+                                          dict_t *xattr)
+{
+        afr_private_t *priv        = NULL;
+        int            i           = 0;
+        int            ret         = -1;
+        void          *pending_raw = NULL;
+        int32_t       *pending     = NULL;
+
+        GF_ASSERT (local);
+        GF_ASSERT (this);
+        GF_ASSERT (xattr);
+
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                ret = dict_get_ptr (xattr, priv->pending_key[i],
+                                    &pending_raw);
+                if (ret != 0) {
+                        continue;
+                }
+                pending = pending_raw;
+
+                if (pending[AFR_METADATA_TRANSACTION]) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "metadata self-heal is pending for %s.",
+                               local->loc.path);
+                        local->self_heal.do_metadata_self_heal = _gf_true;
+                }
+
+                if (pending[AFR_ENTRY_TRANSACTION]) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "entry self-heal is pending for %s.",
+                               local->loc.path);
+                        local->self_heal.do_entry_self_heal = _gf_true;
+                }
+
+                if (pending[AFR_DATA_TRANSACTION]) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "data self-heal is pending for %s.",
+                               local->loc.path);
+                        local->self_heal.do_data_self_heal = _gf_true;
+                }
+        }
+}
+
+void
+afr_lookup_check_set_metadata_split_brain (afr_local_t *local, xlator_t *this)
+{
+        int32_t                  *sources = NULL;
+        afr_private_t            *priv = NULL;
+        int32_t                  subvol_status = 0;
+        int32_t                  *success_children   = NULL;
+        dict_t                   **xattrs = NULL;
+        struct iatt              *bufs = NULL;
+        int32_t                  **pending_matrix = NULL;
+
+        priv = this->private;
+
+        sources = GF_CALLOC (priv->child_count, sizeof (*sources),
+                             gf_afr_mt_int32_t);
+        if (NULL == sources)
+                goto out;
+        success_children = local->cont.lookup.success_children;
+        xattrs = local->cont.lookup.xattrs;
+        bufs = local->cont.lookup.bufs;
+        pending_matrix = local->cont.lookup.pending_matrix;
+        afr_build_sources (this, xattrs, bufs, pending_matrix,
+                           sources, success_children, AFR_METADATA_TRANSACTION,
+                           &subvol_status, _gf_false);
+        if (subvol_status & SPLIT_BRAIN)
+                local->cont.lookup.possible_spb = _gf_true;
+out:
+        GF_FREE (sources);
+}
+
+static void
+afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
+                            struct iatt *buf, struct iatt *lookup_buf)
+{
+        if (PERMISSION_DIFFERS (buf, lookup_buf)) {
+                /* mismatching permissions */
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "permissions differ for %s ", local->loc.path);
+                local->self_heal.do_metadata_self_heal = _gf_true;
+        }
+
+        if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
+                /* mismatching permissions */
+                local->self_heal.do_metadata_self_heal = _gf_true;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "ownership differs for %s ", local->loc.path);
+        }
+
+        if (SIZE_DIFFERS (buf, lookup_buf)
+            && IA_ISREG (buf->ia_type)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "size differs for %s ", local->loc.path);
+                local->self_heal.do_data_self_heal = _gf_true;
+        }
+
+        if (uuid_compare (buf->ia_gfid, lookup_buf->ia_gfid)) {
+                /* mismatching gfid */
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: gfid different on subvolume", local->loc.path);
+        }
+}
+
+static void
+afr_detect_self_heal_by_split_brain_status (afr_local_t *local, xlator_t *this)
+{
+        gf_boolean_t split_brain = _gf_false;
+        afr_self_heal_t *sh = NULL;
+
+        sh = &local->self_heal;
+
+        split_brain = afr_is_split_brain (this, local->cont.lookup.inode);
+        split_brain = split_brain || local->cont.lookup.possible_spb;
+        if ((local->success_count > 0) && split_brain &&
+            IA_ISREG (local->cont.lookup.inode->ia_type)) {
+                sh->force_confirm_spb = _gf_true;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "split brain detected during lookup of %s.",
+                        local->loc.path);
+        }
+}
+
+static void
+afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this)
+{
+        GF_ASSERT (local);
+        GF_ASSERT (this);
+
+        if ((local->success_count > 0) && (local->enoent_count > 0)) {
+                local->self_heal.do_metadata_self_heal = _gf_true;
+                local->self_heal.do_data_self_heal     = _gf_true;
+                local->self_heal.do_entry_self_heal    = _gf_true;
+                local->self_heal.do_gfid_self_heal    = _gf_true;
+                local->self_heal.do_missing_entry_self_heal    = _gf_true;
+                gf_log(this->name, GF_LOG_DEBUG,
+                       "entries are missing in lookup of %s.",
+                       local->loc.path);
+        }
+
+        return;
+}
+
+gf_boolean_t
+afr_can_self_heal_proceed (afr_self_heal_t *sh, afr_private_t *priv)
+{
+        GF_ASSERT (sh);
+        GF_ASSERT (priv);
+
+        if (sh->force_confirm_spb)
+                return _gf_true;
+        return (sh->do_gfid_self_heal
+                || sh->do_missing_entry_self_heal
+                || (afr_data_self_heal_enabled (priv->data_self_heal) &&
+                    sh->do_data_self_heal)
+                || (priv->metadata_self_heal && sh->do_metadata_self_heal)
+                || (priv->entry_self_heal && sh->do_entry_self_heal));
+}
+
+afr_transaction_type
+afr_transaction_type_get (ia_type_t ia_type)
+{
+        afr_transaction_type    type = AFR_METADATA_TRANSACTION;
+
+        GF_ASSERT (ia_type != IA_INVAL);
+
+        if (IA_ISDIR (ia_type)) {
+                type = AFR_ENTRY_TRANSACTION;
+        } else if (IA_ISREG (ia_type)) {
+                type = AFR_DATA_TRANSACTION;
+        }
+        return type;
+}
+
+int
+afr_lookup_select_read_child (afr_local_t *local, xlator_t *this,
+                              int32_t *read_child)
+{
+        ia_type_t               ia_type        = IA_INVAL;
+        int32_t                 source         = -1;
+        int                     ret            = -1;
+        dict_t                  **xattrs       = NULL;
+        int32_t                 *success_children = NULL;
+        afr_transaction_type    type           = AFR_METADATA_TRANSACTION;
+        uuid_t                  *gfid          = NULL;
+
+        GF_ASSERT (local);
+        GF_ASSERT (this);
+        GF_ASSERT (local->success_count > 0);
+
+        success_children = local->cont.lookup.success_children;
+        /*We can take the success_children[0] only because we already
+         *handle the conflicting children other wise, we could select the
+         *read_child based on wrong file type
+         */
+        ia_type = local->cont.lookup.bufs[success_children[0]].ia_type;
+        type = afr_transaction_type_get (ia_type);
+        xattrs = local->cont.lookup.xattrs;
+        gfid = &local->cont.lookup.buf.ia_gfid;
+        source = afr_lookup_select_read_child_by_txn_type (this, local, xattrs,
+                                                           type, *gfid);
+        if (source < 0) {
+                gf_log (this->name, GF_LOG_DEBUG, "failed to select source "
+                        "for %s", local->loc.path);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Source selected as %d for %s",
+                source, local->loc.path);
+        *read_child = source;
+        ret = 0;
+out:
+        return ret;
+}
+
+static inline gf_boolean_t
+afr_is_transaction_running (afr_local_t *local)
+{
+        GF_ASSERT (local->fop == GF_FOP_LOOKUP);
+        return ((local->inodelk_count > 0) || (local->entrylk_count > 0));
+}
+
+void
+afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t background, ia_type_t ia_type, char *reason,
+                      void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
+                                                   xlator_t *this),
+                      int (*unwind) (call_frame_t *frame, xlator_t *this,
+                                     int32_t op_ret, int32_t op_errno,
+                                     int32_t sh_failed))
+{
+        afr_local_t             *local = NULL;
+        char                    sh_type_str[256] = {0,};
+        char                    *bg = "";
+
+        GF_ASSERT (frame);
+        GF_ASSERT (this);
+        GF_ASSERT (inode);
+        GF_ASSERT (ia_type != IA_INVAL);
+
+        local = frame->local;
+        local->self_heal.background = background;
+        local->self_heal.type       = ia_type;
+        local->self_heal.unwind     = unwind;
+        local->self_heal.gfid_sh_success_cbk     = gfid_sh_success_cbk;
+
+        afr_self_heal_type_str_get (&local->self_heal,
+                                    sh_type_str,
+                                    sizeof (sh_type_str));
+
+        if (background)
+                bg = "background";
+        gf_log (this->name, GF_LOG_DEBUG,
+                "%s %s self-heal triggered. path: %s, reason: %s", bg,
+                sh_type_str, local->loc.path, reason);
+
+        afr_self_heal (frame, this, inode);
+}
+
+unsigned int
+afr_gfid_missing_count (const char *xlator_name, int32_t *success_children,
+                        struct iatt *bufs, unsigned int child_count,
+                        const char *path)
+{
+        unsigned int    gfid_miss_count   = 0;
+        int             i              = 0;
+        struct iatt     *child1        = NULL;
+
+        for (i = 0; i < child_count; i++) {
+                if (success_children[i] == -1)
+                        break;
+                child1 = &bufs[success_children[i]];
+                if (uuid_is_null (child1->ia_gfid)) {
+                        gf_log (xlator_name, GF_LOG_DEBUG, "%s: gfid is null"
+                                " on subvolume %d", path, success_children[i]);
+                        gfid_miss_count++;
+                }
+        }
+
+        return gfid_miss_count;
+}
+
+static int
+afr_lookup_gfid_missing_count (afr_local_t *local, xlator_t *this)
+{
+        int32_t         *success_children = NULL;
+        afr_private_t   *priv          = NULL;
+        struct iatt     *bufs          = NULL;
+        int             miss_count     = 0;
+
+        priv = this->private;
+        bufs = local->cont.lookup.bufs;
+        success_children = local->cont.lookup.success_children;
+
+        miss_count =  afr_gfid_missing_count (this->name, success_children,
+                                              bufs, priv->child_count,
+                                              local->loc.path);
+        return miss_count;
+}
+
+gf_boolean_t
+afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
+                        unsigned int child_count, const char *path,
+                        const char *xlator_name)
+{
+        gf_boolean_t    conflicting    = _gf_false;
+        int             i              = 0;
+        struct iatt     *child1        = NULL;
+        struct iatt     *child2        = NULL;
+        uuid_t          *gfid          = NULL;
+
+        for (i = 0; i < child_count; i++) {
+                if (success_children[i] == -1)
+                        break;
+                child1 = &bufs[success_children[i]];
+                if ((!gfid) && (!uuid_is_null (child1->ia_gfid)))
+                        gfid = &child1->ia_gfid;
+
+                if (i == 0)
+                        continue;
+
+                child2 = &bufs[success_children[i-1]];
+                if (FILETYPE_DIFFERS (child1, child2)) {
+                        gf_log (xlator_name, GF_LOG_DEBUG, "%s: filetype "
+                                "differs on subvolumes (%d, %d)", path,
+                                success_children[i-1], success_children[i]);
+                        conflicting = _gf_true;
+                        goto out;
+                }
+                if (!gfid || uuid_is_null (child1->ia_gfid))
+                        continue;
+                if (uuid_compare (*gfid, child1->ia_gfid)) {
+                       gf_log (xlator_name, GF_LOG_DEBUG, "%s: gfid differs"
+                               " on subvolume %d", path, success_children[i]);
+                       conflicting = _gf_true;
+                       goto out;
+                }
+        }
+out:
+        return conflicting;
+}
+
+/* afr_update_gfid_from_iatts: This function should be called only if the
+ * iatts are not conflicting.
+ */
+void
+afr_update_gfid_from_iatts (uuid_t uuid, struct iatt *bufs,
+                            int32_t *success_children, unsigned int child_count)
+{
+        uuid_t          *gfid = NULL;
+        int             i = 0;
+        int             child = 0;
+
+        for (i = 0; i < child_count; i++) {
+                child = success_children[i];
+                if (child == -1)
+                        break;
+                if ((!gfid) && (!uuid_is_null (bufs[child].ia_gfid))) {
+                        gfid = &bufs[child].ia_gfid;
+                } else if (gfid && (!uuid_is_null (bufs[child].ia_gfid))) {
+                        if (uuid_compare (*gfid, bufs[child].ia_gfid)) {
+                                GF_ASSERT (0);
+                                goto out;
+                        }
+                }
+        }
+        if (gfid && (!uuid_is_null (*gfid)))
+                uuid_copy (uuid, *gfid);
+out:
+        return;
+}
+
+static gf_boolean_t
+afr_lookup_conflicting_entries (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t           *priv = NULL;
+        gf_boolean_t            conflict = _gf_false;
+
+        priv = this->private;
+        conflict =  afr_conflicting_iattrs (local->cont.lookup.bufs,
+                                            local->cont.lookup.success_children,
+                                            priv->child_count, local->loc.path,
+                                            this->name);
+        return conflict;
+}
+
+gf_boolean_t
+afr_open_only_data_self_heal (char *data_self_heal)
+{
+        return !strcmp (data_self_heal, "open");
+}
+
+gf_boolean_t
+afr_data_self_heal_enabled (char *data_self_heal)
+{
+        gf_boolean_t    enabled = _gf_false;
+
+        if (gf_string2boolean (data_self_heal, &enabled) == -1) {
+                enabled = !strcmp (data_self_heal, "open");
+                GF_ASSERT (enabled);
+        }
+
+        return enabled;
+}
+
+static void
+afr_lookup_set_self_heal_params (afr_local_t *local, xlator_t *this)
+{
+        int                     i = 0;
+        struct iatt             *bufs = NULL;
+        dict_t                  **xattr = NULL;
+        afr_private_t           *priv = NULL;
+        int32_t                 child1 = -1;
+        int32_t                 child2 = -1;
+        afr_self_heal_t         *sh = NULL;
+
+        priv  = this->private;
+        sh = &local->self_heal;
+
+        afr_detect_self_heal_by_lookup_status (local, this);
+
+        if (afr_lookup_gfid_missing_count (local, this))
+                local->self_heal.do_gfid_self_heal    = _gf_true;
+
+        if (_gf_true == afr_lookup_conflicting_entries (local, this))
+                local->self_heal.do_missing_entry_self_heal    = _gf_true;
+        else
+                afr_update_gfid_from_iatts (local->self_heal.sh_gfid_req,
+                                            local->cont.lookup.bufs,
+                                            local->cont.lookup.success_children,
+                                            priv->child_count);
+
+        bufs = local->cont.lookup.bufs;
+        for (i = 1; i < local->success_count; i++) {
+                child1 = local->cont.lookup.success_children[i-1];
+                child2 = local->cont.lookup.success_children[i];
+                afr_detect_self_heal_by_iatt (local, this,
+                                              &bufs[child1], &bufs[child2]);
+        }
+
+        xattr = local->cont.lookup.xattrs;
+        for (i = 0; i < local->success_count; i++) {
+                child1 = local->cont.lookup.success_children[i];
+                afr_lookup_set_self_heal_params_by_xattr (local, this,
+                                                          xattr[child1]);
+        }
+        if (afr_open_only_data_self_heal (priv->data_self_heal))
+                sh->do_data_self_heal = _gf_false;
+        if (sh->do_metadata_self_heal)
+                afr_lookup_check_set_metadata_split_brain (local, this);
+        afr_detect_self_heal_by_split_brain_status (local, this);
+}
+
+int
+afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             int32_t sh_failed)
+{
+        afr_local_t *local = NULL;
+        int         ret    = -1;
+        dict_t      *xattr = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                local->op_ret = -1;
+		local->op_errno = afr_most_important_error(local->op_errno,
+							   op_errno, _gf_true);
+
+                goto out;
+        } else {
+                local->op_ret = 0;
+        }
+
+        afr_lookup_done_success_action (frame, this, _gf_true);
+        xattr = local->cont.lookup.xattr;
+        if (xattr) {
+                ret = dict_set_int32 (xattr, "sh-failed", sh_failed);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                                "sh-failed to %d", local->loc.path, sh_failed);
+
+                if (local->self_heal.actual_sh_started == _gf_true &&
+                    sh_failed == 0) {
+                        ret = dict_set_int32 (xattr, "actual-sh-done", 1);
+                        if (ret)
+                                gf_log(this->name, GF_LOG_ERROR, "%s: Failed to"
+                                       " set actual-sh-done to %d",
+                                       local->loc.path,
+                                       local->self_heal.actual_sh_started);
+                }
+        }
+out:
+        AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+                          local->cont.lookup.inode, &local->cont.lookup.buf,
+                          local->cont.lookup.xattr,
+                          &local->cont.lookup.postparent);
+
+        return 0;
+}
+
+//TODO: At the moment only lookup needs this, so not doing any checks, in the
+// future we will have to do fop specific operations
+void
+afr_post_gfid_sh_success (call_frame_t *sh_frame, xlator_t *this)
+{
+        afr_local_t             *local = NULL;
+        afr_local_t             *sh_local = NULL;
+        afr_private_t           *priv = NULL;
+        afr_self_heal_t         *sh = NULL;
+        int                     i = 0;
+        struct iatt             *lookup_bufs = NULL;
+        struct iatt             *lookup_parentbufs = NULL;
+
+        sh_local = sh_frame->local;
+        sh       = &sh_local->self_heal;
+        local = sh->orig_frame->local;
+        lookup_bufs = local->cont.lookup.bufs;
+        lookup_parentbufs = local->cont.lookup.postparents;
+        priv = this->private;
+
+        memcpy (lookup_bufs, sh->buf, priv->child_count * sizeof (*sh->buf));
+        memcpy (lookup_parentbufs, sh->parentbufs,
+                priv->child_count * sizeof (*sh->parentbufs));
+
+        afr_reset_xattr (local->cont.lookup.xattrs, priv->child_count);
+        if (local->cont.lookup.xattr) {
+                dict_unref (local->cont.lookup.xattr);
+                local->cont.lookup.xattr = NULL;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->xattr[i])
+                        local->cont.lookup.xattrs[i] = dict_ref (sh->xattr[i]);
+        }
+
+        afr_reset_children (local->cont.lookup.success_children,
+                            priv->child_count);
+        afr_children_copy (local->cont.lookup.success_children,
+                           sh->fresh_children, priv->child_count);
+}
+
+static void
+afr_lookup_perform_self_heal (call_frame_t *frame, xlator_t *this,
+                              gf_boolean_t *sh_launched)
+{
+        unsigned int         up_count = 0;
+        afr_private_t       *priv    = NULL;
+        afr_local_t         *local   = NULL;
+        char                *reason  = NULL;
+
+        GF_ASSERT (sh_launched);
+        *sh_launched = _gf_false;
+        priv         = this->private;
+        local        = frame->local;
+
+        up_count  = afr_up_children_count (local->child_up, priv->child_count);
+        if (up_count == 1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Only 1 child up - do not attempt to detect self heal");
+                goto out;
+        }
+
+        afr_lookup_set_self_heal_params (local, this);
+        if (afr_can_self_heal_proceed (&local->self_heal, priv)) {
+                if  (afr_is_transaction_running (local) &&
+                     (!local->allow_sh_for_running_transaction))
+                        goto out;
+
+                reason = "lookup detected pending operations";
+                afr_launch_self_heal (frame, this, local->cont.lookup.inode,
+                                      _gf_true, local->cont.lookup.buf.ia_type,
+                                      reason, afr_post_gfid_sh_success,
+                                      afr_self_heal_lookup_unwind);
+                *sh_launched = _gf_true;
+        }
+out:
+        return;
+}
+
+void
+afr_get_fresh_children (int32_t *success_children, int32_t *sources,
+                        int32_t *fresh_children, unsigned int child_count)
+{
+        unsigned int i = 0;
+        unsigned int j = 0;
+
+        GF_ASSERT (success_children);
+        GF_ASSERT (sources);
+        GF_ASSERT (fresh_children);
+
+        afr_reset_children (fresh_children, child_count);
+        for (i = 0; i < child_count; i++) {
+                if (success_children[i] == -1)
+                        break;
+                if (afr_is_read_child (success_children, sources, child_count,
+                                       success_children[i])) {
+                        fresh_children[j] = success_children[i];
+                        j++;
+                }
+        }
+}
+
+static int
+afr_lookup_set_read_ctx (afr_local_t *local, xlator_t *this, int32_t read_child)
+{
+        afr_private_t           *priv = NULL;
+
+        GF_ASSERT (read_child >= 0);
+
+        priv = this->private;
+        afr_get_fresh_children (local->cont.lookup.success_children,
+                                local->cont.lookup.sources,
+                                local->fresh_children, priv->child_count);
+        afr_inode_set_read_ctx (this, local->cont.lookup.inode, read_child,
+                                local->fresh_children);
+
+        return 0;
+}
+
+int
+afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
+                                gf_boolean_t fail_conflict)
+{
+        int32_t             read_child = -1;
+        int32_t             ret        = -1;
+        afr_local_t         *local     = NULL;
+        gf_boolean_t        fresh_lookup = _gf_false;
+
+        local   = frame->local;
+        fresh_lookup = local->cont.lookup.fresh_lookup;
+
+        if (local->loc.parent == NULL)
+                fail_conflict = _gf_true;
+
+        if (afr_lookup_conflicting_entries (local, this)) {
+                if (fail_conflict == _gf_false)
+                        ret = 0;
+                goto out;
+        }
+
+        ret = afr_lookup_select_read_child (local, this, &read_child);
+        if (!afr_is_transaction_running (local) || fresh_lookup) {
+                if (read_child < 0)
+                        goto out;
+
+                ret = afr_lookup_set_read_ctx (local, this, read_child);
+                if (ret)
+                        goto out;
+        }
+
+        ret = afr_lookup_build_response_params (local, this);
+        if (ret)
+                goto out;
+        afr_update_loc_gfids (&local->loc,
+                              &local->cont.lookup.buf,
+                              &local->cont.lookup.postparent);
+
+        ret = 0;
+out:
+        if (ret) {
+                local->op_ret = -1;
+                local->op_errno = EIO;
+        }
+        return ret;
+}
+
+int
+afr_lookup_get_latest_subvol (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        int32_t       *success_children = NULL;
+        struct iatt   *bufs = NULL;
+        int           i = 0;
+        int           child = 0;
+        int           lsubvol = -1;
+
+        priv = this->private;
+        success_children = local->cont.lookup.success_children;
+        bufs = local->cont.lookup.bufs;
+        for (i = 0; i < priv->child_count; i++) {
+                child = success_children[i];
+                if (child == -1)
+                        break;
+                if (uuid_is_null (bufs[child].ia_gfid))
+                        continue;
+                if (lsubvol < 0) {
+                        lsubvol = child;
+                } else if (bufs[lsubvol].ia_ctime < bufs[child].ia_ctime) {
+                        lsubvol = child;
+                } else if ((bufs[lsubvol].ia_ctime == bufs[child].ia_ctime) &&
+                  (bufs[lsubvol].ia_ctime_nsec < bufs[child].ia_ctime_nsec)) {
+                        lsubvol = child;
+                }
+        }
+        return lsubvol;
+}
+
+void
+afr_lookup_mark_other_entries_stale (afr_local_t *local, xlator_t *this,
+                                     int subvol)
+{
+        afr_private_t *priv = NULL;
+        int32_t       *success_children = NULL;
+        struct iatt   *bufs = NULL;
+        int           i = 0;
+        int           child = 0;
+
+        priv = this->private;
+        success_children = local->cont.lookup.success_children;
+        bufs = local->cont.lookup.bufs;
+        memcpy (local->fresh_children, success_children,
+                sizeof (*success_children) * priv->child_count);
+        for (i = 0; i < priv->child_count; i++) {
+                child = local->fresh_children[i];
+                if (child == -1)
+                        break;
+                if (child == subvol)
+                        continue;
+                if (uuid_is_null (bufs[child].ia_gfid) &&
+                    (bufs[child].ia_type == bufs[subvol].ia_type))
+                        continue;
+                afr_children_rm_child (success_children, child,
+                                       priv->child_count);
+                local->success_count--;
+        }
+        afr_reset_children (local->fresh_children, priv->child_count);
+}
+
+void
+afr_succeed_lookup_on_latest_iatt (afr_local_t *local, xlator_t *this)
+{
+        int    lsubvol = 0;
+
+        if (!afr_lookup_conflicting_entries (local, this))
+                goto out;
+
+        lsubvol = afr_lookup_get_latest_subvol (local, this);
+        if (lsubvol < 0)
+                goto out;
+        afr_lookup_mark_other_entries_stale (local, this, lsubvol);
+out:
+        return;
+}
+
+gf_boolean_t
+afr_is_entry_possibly_under_creation (afr_local_t *local, xlator_t *this)
+{
+        /*
+         * We need to perform this test in lookup done and treat on going
+         * create/DELETE as ENOENT.
+         * Reason:
+        Multiple clients A, B and C are attempting 'mkdir -p /mnt/a/b/c'
+
+        1 Client A is in the middle of mkdir(/a). It has acquired lock.
+          It has performed mkdir(/a) on one subvol, and second one is still
+          in progress
+        2 Client B performs a lookup, sees directory /a on one,
+          ENOENT on the other, succeeds lookup.
+        3 Client B performs lookup on /a/b on both subvols, both return ENOENT
+          (one subvol because /a/b does not exist, another because /a
+          itself does not exist)
+        4 Client B proceeds to mkdir /a/b. It obtains entrylk on inode=/a with
+          basename=b on one subvol, but fails on other subvol as /a is yet to
+          be created by Client A.
+        5 Client A finishes mkdir of /a on other subvol
+        6 Client C also attempts to create /a/b, lookup returns ENOENT on
+          both subvols.
+        7 Client C tries to obtain entrylk on on inode=/a with basename=b,
+          obtains on one subvol (where B had failed), and waits for B to unlock
+          on other subvol.
+        8 Client B finishes mkdir() on one subvol with GFID-1 and completes
+          transaction and unlocks
+        9 Client C gets the lock on the second subvol, At this stage second
+          subvol already has /a/b created from Client B, but Client C does not
+          check that in the middle of mkdir transaction
+        10 Client C attempts mkdir /a/b on both subvols. It succeeds on
+           ONLY ONE (where Client B could not get lock because of
+           missing parent /a dir) with GFID-2, and gets EEXIST from ONE subvol.
+        This way we have /a/b in GFID mismatch. One subvol got GFID-1 because
+        Client B performed transaction on only one subvol (because entrylk()
+        could not be obtained on second subvol because of missing parent dir --
+        caused by premature/speculative succeeding of lookup() on /a when locks
+        are detected). Other subvol gets GFID-2 from Client C because while
+        it was waiting for entrylk() on both subvols, Client B was in the
+        middle of creating mkdir() on only one subvol, and Client C does not
+        "expect" this when it is between lock() and pre-op()/op() phase of the
+        transaction.
+         */
+	if (local->cont.lookup.parent_entrylk && local->enoent_count)
+		return _gf_true;
+
+	return _gf_false;
+}
+
+
+static void
+afr_lookup_done (call_frame_t *frame, xlator_t *this)
+{
+        int                 unwind = 1;
+        afr_private_t       *priv  = NULL;
+        afr_local_t         *local = NULL;
+        int                 ret = -1;
+        gf_boolean_t        sh_launched = _gf_false;
+        gf_boolean_t        fail_conflict = _gf_false;
+        int                 gfid_miss_count = 0;
+        int                 enotconn_count = 0;
+        int                 up_children_count = 0;
+
+        priv  = this->private;
+        local = frame->local;
+
+	if (afr_is_entry_possibly_under_creation (local, this)) {
+		local->op_ret = -1;
+		local->op_errno = ENOENT;
+		goto unwind;
+	}
+
+        if (local->op_ret < 0)
+                goto unwind;
+
+        if (local->cont.lookup.parent_entrylk && local->success_count > 1)
+                afr_succeed_lookup_on_latest_iatt (local, this);
+
+        gfid_miss_count = afr_lookup_gfid_missing_count (local, this);
+        up_children_count = afr_up_children_count (local->child_up,
+                                                   priv->child_count);
+        enotconn_count = priv->child_count - up_children_count;
+        if ((gfid_miss_count == local->success_count) &&
+            (enotconn_count > 0)) {
+                local->op_ret = -1;
+                local->op_errno = EIO;
+                gf_log (this->name, GF_LOG_ERROR, "Failing lookup for %s, "
+                        "LOOKUP on a file without gfid is not allowed when "
+                        "some of the children are down", local->loc.path);
+                goto unwind;
+        }
+
+        if ((gfid_miss_count == local->success_count) &&
+            uuid_is_null (local->cont.lookup.gfid_req)) {
+                local->op_ret = -1;
+                local->op_errno = ENODATA;
+                gf_log (this->name, GF_LOG_ERROR, "%s: No gfid present",
+                        local->loc.path);
+                goto unwind;
+        }
+
+        if (gfid_miss_count && uuid_is_null (local->cont.lookup.gfid_req))
+                fail_conflict = _gf_true;
+        ret = afr_lookup_done_success_action (frame, this, fail_conflict);
+        if (ret)
+                goto unwind;
+        uuid_copy (local->self_heal.sh_gfid_req, local->cont.lookup.gfid_req);
+
+        afr_lookup_perform_self_heal (frame, this, &sh_launched);
+        if (sh_launched) {
+                unwind = 0;
+                goto unwind;
+        }
+
+ unwind:
+         if (unwind) {
+                 AFR_STACK_UNWIND (lookup, frame, local->op_ret,
+                                   local->op_errno, local->cont.lookup.inode,
+                                   &local->cont.lookup.buf,
+                                   local->cont.lookup.xattr,
+                                   &local->cont.lookup.postparent);
+        }
+}
+
+/*
+ * During a lookup, some errors are more "important" than
+ * others in that they must be given higher priority while
+ * returning to the user.
+ *
+ * The hierarchy is ESTALE > EIO > ENOENT > others
+ */
+int32_t
+afr_most_important_error(int32_t old_errno, int32_t new_errno,
+			 gf_boolean_t eio)
+{
+	if (old_errno == ESTALE || new_errno == ESTALE)
+		return ESTALE;
+	if (eio && (old_errno == EIO || new_errno == EIO))
+		return EIO;
+	if (old_errno == ENOENT || new_errno == ENOENT)
+		return ENOENT;
+
+	return new_errno;
+}
+
+int32_t
+afr_resultant_errno_get (int32_t *children,
+                         int *child_errno, unsigned int child_count)
+{
+        int     i = 0;
+        int32_t op_errno = 0;
+        int     child = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (children) {
+                        child = children[i];
+                        if (child == -1)
+                                break;
+                } else {
+                        child = i;
+                }
+		op_errno = afr_most_important_error(op_errno,
+						    child_errno[child],
+						    _gf_false);
+        }
+        return op_errno;
+}
+
+static void
+afr_lookup_handle_error (afr_local_t *local, int32_t op_ret,  int32_t op_errno)
+{
+        GF_ASSERT (local);
+        if (op_errno == ENOENT)
+                local->enoent_count++;
+
+	local->op_errno = afr_most_important_error(local->op_errno, op_errno,
+						   _gf_false);
+
+        if (local->op_errno == ESTALE) {
+                local->op_ret = -1;
+        }
+}
+
+static void
+afr_set_root_inode_on_first_lookup (afr_local_t *local, xlator_t *this,
+                                    inode_t *inode)
+{
+        afr_private_t           *priv = NULL;
+        GF_ASSERT (inode);
+
+        if (!__is_root_gfid (inode->gfid))
+                goto out;
+        if (!afr_is_fresh_lookup (&local->loc, this))
+                goto out;
+        priv = this->private;
+        if ((priv->first_lookup)) {
+                gf_log (this->name, GF_LOG_INFO, "added root inode");
+                priv->root_inode = inode_ref (inode);
+                priv->first_lookup = 0;
+        }
+out:
+        return;
+}
+
+static void
+afr_lookup_cache_args (afr_local_t *local, int child_index, dict_t *xattr,
+                       struct iatt *buf, struct iatt *postparent)
+{
+        GF_ASSERT (child_index >= 0);
+        local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
+        local->cont.lookup.postparents[child_index] = *postparent;
+        local->cont.lookup.bufs[child_index] = *buf;
+}
+
+static void
+afr_lookup_handle_first_success (afr_local_t *local, xlator_t *this,
+                                 inode_t *inode, struct iatt *buf)
+{
+        local->cont.lookup.inode      = inode_ref (inode);
+        local->cont.lookup.buf        = *buf;
+        afr_set_root_inode_on_first_lookup (local, this, inode);
+}
+
+static int32_t
+afr_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
+{
+        int              ret            = 0;
+        char            *pathinfo       = NULL;
+        gf_boolean_t     is_local        = _gf_false;
+        afr_private_t   *priv           = NULL;
+        int32_t          child_index    = -1;
+
+        if (op_ret != 0) {
+                goto out;
+        }
+
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+        if (ret != 0) {
+                goto out;
+        }
+
+        ret = afr_local_pathinfo (pathinfo, &is_local);
+        if (ret) {
+                goto out;
+        }
+
+        priv = this->private;
+        /*
+         * Note that one local subvolume will override another here.  The only
+         * way to avoid that would be to retain extra information about whether
+         * the previous read_child is local, and it's just not worth it.  Even
+         * the slowest local subvolume is far preferable to a remote one.
+         */
+        if (is_local) {
+                child_index = (int32_t)(long)cookie;
+                gf_log (this->name, GF_LOG_INFO,
+                        "selecting local read_child %s",
+                        priv->children[child_index]->name);
+                priv->read_child = child_index;
+        }
+
+out:
+        STACK_DESTROY(frame->root);
+        return 0;
+}
+
+static void
+afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
+{
+        call_frame_t    *newframe = NULL;
+        loc_t            tmploc = {0,};
+        afr_private_t   *priv = this->private;
+
+        newframe = create_frame(this,this->ctx->pool);
+        if (!newframe) {
+                return;
+        }
+
+        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
+        STACK_WIND_COOKIE (newframe, afr_discovery_cbk,
+                           (void *)(long)child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->getxattr,
+                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
+}
+
+static void
+afr_lookup_handle_success (afr_local_t *local, xlator_t *this, int32_t child_index,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xattr,
+                           struct iatt *postparent)
+{
+        afr_private_t   *priv   = this->private;
+
+        if (local->success_count == 0) {
+                if (local->op_errno != ESTALE) {
+                        local->op_ret = op_ret;
+                        local->op_errno = 0;
+                }
+                afr_lookup_handle_first_success (local, this, inode, buf);
+        }
+        afr_lookup_update_lk_counts (local, this,
+                                     child_index, xattr);
+
+        afr_lookup_cache_args (local, child_index, xattr,
+                               buf, postparent);
+
+        if (local->do_discovery && (priv->read_child == (-1))) {
+                afr_attempt_local_discovery(this,child_index);
+        }
+
+        local->cont.lookup.success_children[local->success_count] = child_index;
+        local->success_count++;
+}
+
+int
+afr_lookup_cbk (call_frame_t *frame, void *cookie,
+                xlator_t *this,  int32_t op_ret,  int32_t op_errno,
+                inode_t *inode,   struct iatt *buf, dict_t *xattr,
+                struct iatt *postparent)
+{
+        afr_local_t *   local = NULL;
+        int             call_count      = -1;
+        int             child_index     = -1;
+
+         child_index = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                if (op_ret == -1) {
+                        afr_lookup_handle_error (local, op_ret, op_errno);
+                        goto unlock;
+                }
+                afr_lookup_handle_success (local, this, child_index, op_ret,
+                                           op_errno, inode, buf, xattr,
+                                           postparent);
+
+         }
+unlock:
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+               afr_lookup_done (frame, this);
+        }
+
+         return 0;
+}
+
+int
+afr_lookup_cont_init (afr_local_t *local, unsigned int child_count)
+{
+        int               ret            = -ENOMEM;
+        struct iatt       *iatts         = NULL;
+        int32_t           *success_children = NULL;
+        int32_t           *sources       = NULL;
+        int32_t           **pending_matrix = NULL;
+
+        GF_ASSERT (local);
+        local->cont.lookup.xattrs = GF_CALLOC (child_count,
+                                               sizeof (*local->cont.lookup.xattr),
+                                               gf_afr_mt_dict_t);
+        if (NULL == local->cont.lookup.xattrs)
+                goto out;
+
+        iatts = GF_CALLOC (child_count, sizeof (*iatts), gf_afr_mt_iatt);
+        if (NULL == iatts)
+                goto out;
+        local->cont.lookup.postparents = iatts;
+
+        iatts = GF_CALLOC (child_count, sizeof (*iatts), gf_afr_mt_iatt);
+        if (NULL == iatts)
+                goto out;
+        local->cont.lookup.bufs = iatts;
+
+        success_children = afr_children_create (child_count);
+        if (NULL == success_children)
+                goto out;
+        local->cont.lookup.success_children = success_children;
+
+        local->fresh_children = afr_children_create (child_count);
+        if (NULL == local->fresh_children)
+                goto out;
+
+        sources = GF_CALLOC (sizeof (*sources), child_count, gf_afr_mt_int32_t);
+        if (NULL == sources)
+                goto out;
+        local->cont.lookup.sources = sources;
+
+        pending_matrix = afr_matrix_create (child_count, child_count);
+        if (NULL == pending_matrix)
+                goto out;
+        local->cont.lookup.pending_matrix = pending_matrix;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+afr_lookup (call_frame_t *frame, xlator_t *this,
+            loc_t *loc, dict_t *xattr_req)
+{
+        afr_private_t    *priv           = NULL;
+        afr_local_t      *local          = NULL;
+        void              *gfid_req      = NULL;
+        int               ret            = -1;
+        int               i              = 0;
+        int               call_count     = 0;
+        uint64_t          ctx            = 0;
+        int32_t           op_errno       = 0;
+        int               allow_sh       = 0;
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
+
+        local->op_ret = -1;
+
+        frame->local = local;
+        local->fop = GF_FOP_LOOKUP;
+
+        loc_copy (&local->loc, loc);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        if (local->loc.path &&
+            (strcmp (local->loc.path, "/" GF_REPLICATE_TRASH_DIR) == 0)) {
+                op_errno = EPERM;
+                ret = -1;
+                goto out;
+        }
+
+        ret = inode_ctx_get (local->loc.inode, this, &ctx);
+        if (ret == 0) {
+                /* lookup is a revalidate */
+
+                local->read_child_index = afr_inode_get_read_ctx (this,
+                                                               local->loc.inode,
+                                                               NULL);
+        } else {
+                LOCK (&priv->read_child_lock);
+                {
+                        if (priv->hash_mode) {
+                                local->read_child_index = -1;
+                        }
+                        else {
+                                local->read_child_index =
+                                        (++priv->read_child_rr) %
+                                        (priv->child_count);
+                        }
+                }
+                UNLOCK (&priv->read_child_lock);
+                local->cont.lookup.fresh_lookup = _gf_true;
+        }
+
+        local->child_up = memdup (priv->child_up,
+                                  sizeof (*local->child_up) * priv->child_count);
+        if (NULL == local->child_up) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        ret = afr_lookup_cont_init (local, priv->child_count);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
+
+        local->call_count = afr_up_children_count (local->child_up,
+                                                   priv->child_count);
+        call_count = local->call_count;
+        if (local->call_count == 0) {
+                ret      = -1;
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        /* By default assume ENOTCONN. On success it will be set to 0. */
+        local->op_errno = ENOTCONN;
+
+        ret = dict_get_int32 (xattr_req, "allow-sh-for-running-transaction",
+                              &allow_sh);
+        dict_del (xattr_req, "allow-sh-for-running-transaction");
+        local->allow_sh_for_running_transaction = allow_sh;
+
+        ret = afr_lookup_xattr_req_prepare (local, this, xattr_req, &local->loc,
+                                            &gfid_req);
+        if (ret) {
+                local->op_errno = -ret;
+                goto out;
+        }
+        afr_lookup_save_gfid (local->cont.lookup.gfid_req, gfid_req,
+                              &local->loc);
+        local->fop = GF_FOP_LOOKUP;
+        if (priv->choose_local && !priv->did_discovery) {
+                if (gfid_req && __is_root_gfid(gfid_req)) {
+                        local->do_discovery = _gf_true;
+                        priv->did_discovery = _gf_true;
+                }
+        }
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_lookup_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->lookup,
+                                           &local->loc, local->xattr_req);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                AFR_STACK_UNWIND (lookup, frame, -1, op_errno,
+                                  NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+/* {{{ open */
+
+int
+__afr_fd_ctx_set (xlator_t *this, fd_t *fd)
+{
+        afr_private_t * priv   = NULL;
+        int             ret    = -1;
+        uint64_t        ctx    = 0;
+        afr_fd_ctx_t *  fd_ctx = NULL;
+
+        VALIDATE_OR_GOTO (this->private, out);
+        VALIDATE_OR_GOTO (fd, out);
+
+        priv = this->private;
+
+        ret = __fd_ctx_get (fd, this, &ctx);
+
+        if (ret == 0)
+                goto out;
+
+        fd_ctx = GF_CALLOC (1, sizeof (afr_fd_ctx_t),
+                            gf_afr_mt_afr_fd_ctx_t);
+        if (!fd_ctx) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        fd_ctx->pre_op_done = GF_CALLOC (sizeof (*fd_ctx->pre_op_done),
+                                         priv->child_count,
+                                         gf_afr_mt_char);
+        if (!fd_ctx->pre_op_done) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        fd_ctx->pre_op_piggyback = GF_CALLOC (sizeof (*fd_ctx->pre_op_piggyback),
+                                              priv->child_count,
+                                              gf_afr_mt_char);
+        if (!fd_ctx->pre_op_piggyback) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
+                                       priv->child_count,
+                                       gf_afr_mt_int32_t);
+        if (!fd_ctx->opened_on) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        fd_ctx->lock_piggyback = GF_CALLOC (sizeof (*fd_ctx->lock_piggyback),
+                                            priv->child_count,
+                                            gf_afr_mt_char);
+        if (!fd_ctx->lock_piggyback) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        fd_ctx->lock_acquired = GF_CALLOC (sizeof (*fd_ctx->lock_acquired),
+                                           priv->child_count,
+                                           gf_afr_mt_char);
+        if (!fd_ctx->lock_acquired) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        fd_ctx->up_count   = priv->up_count;
+        fd_ctx->down_count = priv->down_count;
+
+        fd_ctx->locked_on = GF_CALLOC (sizeof (*fd_ctx->locked_on),
+                                       priv->child_count,
+                                       gf_afr_mt_char);
+        if (!fd_ctx->locked_on) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+	pthread_mutex_init (&fd_ctx->delay_lock, NULL);
+        INIT_LIST_HEAD (&fd_ctx->entries);
+        fd_ctx->call_child = -1;
+
+        INIT_LIST_HEAD (&fd_ctx->eager_locked);
+
+        ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to set fd ctx (%p)", fd);
+out:
+        return ret;
+}
+
+
+int
+afr_fd_ctx_set (xlator_t *this, fd_t *fd)
+{
+        int ret = -1;
+
+        LOCK (&fd->lock);
+        {
+                ret = __afr_fd_ctx_set (this, fd);
+        }
+        UNLOCK (&fd->lock);
+
+        return ret;
+}
+
+/* {{{ flush */
+
+int
+afr_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        int call_count  = -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret != -1) {
+                        if (local->success_count == 0) {
+                                local->op_ret = op_ret;
+                        }
+                        local->success_count++;
+                }
+
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+	call_count = afr_frame_return (frame);
+
+	if (call_count == 0)
+		AFR_STACK_UNWIND(flush, frame, local->op_ret,
+				 local->op_errno, NULL);
+
+        return 0;
+}
+
+static int
+afr_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        int           i      = 0;
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int call_count       = -1;
+
+        priv = this->private;
+        local = frame->local;
+        call_count = local->call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_flush_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->flush,
+                                           local->fd, NULL);
+                        if (!--call_count)
+                                break;
+
+                }
+        }
+
+        return 0;
+}
+
+int
+afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+        afr_local_t   *local = NULL;
+        call_stub_t   *stub = NULL;
+        int            ret        = -1;
+        int            op_errno   = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+	AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+	local = frame->local;
+
+	ret = afr_local_init(local, priv, &op_errno);
+	if (ret < 0)
+		goto out;
+
+	local->fd = fd_ref(fd);
+        stub = fop_flush_stub (frame, afr_flush_wrapper, fd, xdata);
+        if (!stub) {
+                ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        afr_delayed_changelog_wake_resume (this, fd, stub);
+	ret = 0;
+
+out:
+	if (ret < 0)
+		AFR_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+
+        return 0;
+}
+
+/* }}} */
+
+
+int
+afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
+{
+        uint64_t        ctx = 0;
+        afr_fd_ctx_t    *fd_ctx = NULL;
+        int             ret = 0;
+
+        ret = fd_ctx_get (fd, this, &ctx);
+        if (ret < 0)
+                goto out;
+
+        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+
+        if (fd_ctx) {
+                GF_FREE (fd_ctx->pre_op_done);
+
+                GF_FREE (fd_ctx->opened_on);
+
+                GF_FREE (fd_ctx->locked_on);
+
+                GF_FREE (fd_ctx->pre_op_piggyback);
+                GF_FREE (fd_ctx->lock_piggyback);
+
+                GF_FREE (fd_ctx->lock_acquired);
+
+		pthread_mutex_destroy (&fd_ctx->delay_lock);
+
+                GF_FREE (fd_ctx);
+        }
+
+out:
+        return 0;
+}
+
+
+int
+afr_release (xlator_t *this, fd_t *fd)
+{
+        afr_locked_fd_t *locked_fd = NULL;
+        afr_locked_fd_t *tmp       = NULL;
+        afr_private_t   *priv      = NULL;
+
+        priv = this->private;
+
+        afr_cleanup_fd_ctx (this, fd);
+
+        list_for_each_entry_safe (locked_fd, tmp, &priv->saved_fds,
+                                  list) {
+
+                if (locked_fd->fd == fd) {
+                        list_del_init (&locked_fd->list);
+                        GF_FREE (locked_fd);
+                }
+
+        }
+
+        return 0;
+}
+
+
+/* {{{ fsync */
+
+int
+afr_fsync_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+        return 0;
+}
+
+int
+afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int call_count = -1;
+        int child_index = (long) cookie;
+        int read_child  = 0;
+	call_stub_t *stub = NULL;
+
+        local = frame->local;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                if (op_ret == 0) {
+                        local->op_ret = 0;
+
+                        if (local->success_count == 0) {
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
+                        }
+
+                        if (child_index == read_child) {
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
+                        }
+
+                        local->success_count++;
+                }
+
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+		/* Make a stub out of the frame, and register it
+		   with the waking up post-op. When the call-stub resumes,
+		   we are guaranteed that there was no post-op pending
+		   (i.e changelogs were unset in the server). This is an
+		   essential "guarantee", that fsync() returns only after
+		   completely finishing EVERYTHING, including the delayed
+		   post-op. This guarantee is expected by FUSE graph switching
+		   for example.
+		*/
+		stub = fop_fsync_cbk_stub (frame, afr_fsync_unwind_cbk,
+                                           local->op_ret, local->op_errno,
+                                           &local->cont.inode_wfop.prebuf,
+                                           &local->cont.inode_wfop.postbuf,
+                                           xdata);
+		if (!stub) {
+			AFR_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
+			return 0;
+		}
+
+		/* If no new unstable writes happened between the
+		   time we cleared the unstable write witness flag in afr_fsync
+		   and now, calling afr_delayed_changelog_wake_up() should
+		   wake up and skip over the fsync phase and go straight to
+		   afr_changelog_post_op_now()
+		*/
+		afr_delayed_changelog_wake_resume (this, local->fd, stub);
+        }
+
+        return 0;
+}
+
+
+int
+afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
+           int32_t datasync, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int ret = -1;
+        int i = 0;
+        int32_t call_count = 0;
+        int32_t op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count = local->call_count;
+
+        local->fd             = fd_ref (fd);
+
+	if (afr_fd_has_witnessed_unstable_write (this, fd)) {
+		/* don't care. we only wanted to CLEAR the bit */
+	}
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_fsync_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fsync,
+                                           fd, datasync, xdata);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+/* }}} */
+
+/* {{{ fsync */
+
+int32_t
+afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int call_count = -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == 0)
+                        local->op_ret = 0;
+
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
+                                  local->op_errno, xdata);
+
+        return 0;
+}
+
+
+int32_t
+afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              int32_t datasync, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int ret = -1;
+        int i = 0;
+        int32_t call_count = 0;
+        int32_t op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count = local->call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND (frame, afr_fsyncdir_cbk,
+                                    priv->children[i],
+                                    priv->children[i]->fops->fsyncdir,
+                                    fd, datasync, xdata);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+        return 0;
+}
+
+/* }}} */
+
+/* {{{ xattrop */
+
+int32_t
+afr_xattrop_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno,
+                 dict_t *xattr, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int call_count = -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == 0) {
+                        if (!local->cont.xattrop.xattr)
+                                local->cont.xattrop.xattr = dict_ref (xattr);
+                        local->op_ret = 0;
+                }
+
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
+                local->cont.xattrop.xattr, xdata);
+
+        return 0;
+}
+
+
+int32_t
+afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local  = NULL;
+        int ret = -1;
+        int i = 0;
+        int32_t call_count = 0;
+        int32_t op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count = local->call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND (frame, afr_xattrop_cbk,
+                                    priv->children[i],
+                                    priv->children[i]->fops->xattrop,
+                                    loc, optype, xattr, xdata);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+/* }}} */
+
+/* {{{ fxattrop */
+
+int32_t
+afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  dict_t *xattr, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+
+        int call_count = -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == 0) {
+                        if (!local->cont.fxattrop.xattr)
+                                local->cont.fxattrop.xattr = dict_ref (xattr);
+
+                        local->op_ret = 0;
+                }
+
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
+                                  local->cont.fxattrop.xattr, xdata);
+
+        return 0;
+}
+
+
+int32_t
+afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local  = NULL;
+        int ret = -1;
+        int i = 0;
+        int32_t call_count = 0;
+        int32_t op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count = local->call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND (frame, afr_fxattrop_cbk,
+                                    priv->children[i],
+                                    priv->children[i]->fops->fxattrop,
+                                    fd, optype, xattr, xdata);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+/* }}} */
+
+
+int32_t
+afr_inodelk_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+
+{
+        afr_local_t *local = NULL;
+        int call_count = -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == 0)
+                        local->op_ret = 0;
+
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
+                                  local->op_errno, xdata);
+
+        return 0;
+}
+
+
+int32_t
+afr_inodelk (call_frame_t *frame, xlator_t *this,
+             const char *volume, loc_t *loc, int32_t cmd,
+             struct gf_flock *flock, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local  = NULL;
+        int ret = -1;
+        int i = 0;
+        int32_t call_count = 0;
+        int32_t op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count = local->call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND (frame, afr_inodelk_cbk,
+                                    priv->children[i],
+                                    priv->children[i]->fops->inodelk,
+                                    volume, loc, cmd, flock, xdata);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+        return 0;
+}
+
+
+int32_t
+afr_finodelk_cbk (call_frame_t *frame, void *cookie,
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  dict_t *xdata)
+
+{
+        afr_local_t *local = NULL;
+        int call_count = -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == 0)
+                        local->op_ret = 0;
+
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                AFR_STACK_UNWIND (finodelk, frame, local->op_ret,
+                                  local->op_errno, xdata);
+
+        return 0;
+}
+
+
+int32_t
+afr_finodelk (call_frame_t *frame, xlator_t *this,
+              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+              dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local  = NULL;
+        int ret = -1;
+        int i = 0;
+        int32_t call_count = 0;
+        int32_t op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count = local->call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND (frame, afr_finodelk_cbk,
+                                    priv->children[i],
+                                    priv->children[i]->fops->finodelk,
+                                    volume, fd, cmd, flock, xdata);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+        return 0;
+}
+
+
+int32_t
+afr_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int call_count = -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == 0)
+                        local->op_ret = 0;
+
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                AFR_STACK_UNWIND (entrylk, frame, local->op_ret,
+                                  local->op_errno, xdata);
+
+        return 0;
+}
+
+
+int32_t
+afr_entrylk (call_frame_t *frame, xlator_t *this,
+             const char *volume, loc_t *loc,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local  = NULL;
+        int ret = -1;
+        int i = 0;
+        int32_t call_count = 0;
+        int32_t op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count = local->call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND (frame, afr_entrylk_cbk,
+                                    priv->children[i],
+                                    priv->children[i]->fops->entrylk,
+                                    volume, loc, basename, cmd, type, xdata);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+        return 0;
+}
+
+
+
+int32_t
+afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
+                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+
+{
+        afr_local_t *local = NULL;
+        int call_count = -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == 0)
+                        local->op_ret = 0;
+
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                AFR_STACK_UNWIND (fentrylk, frame, local->op_ret,
+                                  local->op_errno, xdata);
+
+        return 0;
+}
+
+
+int32_t
+afr_fentrylk (call_frame_t *frame, xlator_t *this,
+              const char *volume, fd_t *fd,
+              const char *basename, entrylk_cmd cmd,
+              entrylk_type type, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local  = NULL;
+        int ret = -1;
+        int i = 0;
+        int32_t call_count = 0;
+        int32_t op_errno = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count = local->call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND (frame, afr_fentrylk_cbk,
+                                    priv->children[i],
+                                    priv->children[i]->fops->fentrylk,
+                                    volume, fd, basename, cmd, type, xdata);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+        return 0;
+}
+
+int32_t
+afr_statfs_cbk (call_frame_t *frame, void *cookie,
+                xlator_t *this, int32_t op_ret, int32_t op_errno,
+                struct statvfs *statvfs, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int call_count = 0;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                if (op_ret == 0) {
+                        local->op_ret   = op_ret;
+
+                        if (local->cont.statfs.buf_set) {
+                                if (statvfs->f_bavail < local->cont.statfs.buf.f_bavail)
+                                        local->cont.statfs.buf = *statvfs;
+                        } else {
+                                local->cont.statfs.buf = *statvfs;
+                                local->cont.statfs.buf_set = 1;
+                        }
+                }
+
+                if (op_ret == -1)
+                        local->op_errno = op_errno;
+
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
+                                  &local->cont.statfs.buf, xdata);
+
+        return 0;
+}
+
+
+int32_t
+afr_statfs (call_frame_t *frame, xlator_t *this,
+            loc_t *loc, dict_t *xdata)
+{
+        afr_private_t *  priv        = NULL;
+        int              child_count = 0;
+        afr_local_t   *  local       = NULL;
+        int              i           = 0;
+        int              ret = -1;
+        int              call_count = 0;
+        int32_t          op_errno    = 0;
+
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+        VALIDATE_OR_GOTO (loc, out);
+
+        priv = this->private;
+        child_count = priv->child_count;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count = local->call_count;
+
+        for (i = 0; i < child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND (frame, afr_statfs_cbk,
+                                    priv->children[i],
+                                    priv->children[i]->fops->statfs,
+                                    loc, xdata);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                   dict_t *xdata)
+{
+        afr_local_t * local = NULL;
+        int call_count = -1;
+
+        local = frame->local;
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
+                                  lock, xdata);
+
+        return 0;
+}
+
+
+int32_t
+afr_lk_unlock (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   * local = NULL;
+        afr_private_t * priv  = NULL;
+        int i = 0;
+        int call_count = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        call_count = afr_locked_nodes_count (local->cont.lk.locked_nodes,
+                                             priv->child_count);
+
+        if (call_count == 0) {
+                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
+                                  &local->cont.lk.ret_flock, NULL);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        local->cont.lk.user_flock.l_type = F_UNLCK;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->cont.lk.locked_nodes[i]) {
+                        STACK_WIND (frame, afr_lk_unlock_cbk,
+                                    priv->children[i],
+                                    priv->children[i]->fops->lk,
+                                    local->fd, F_SETLK,
+                                    &local->cont.lk.user_flock, NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+
+int32_t
+afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+            int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int child_index = -1;
+/*        int            ret  = 0; */
+
+
+        local = frame->local;
+        priv  = this->private;
+
+        child_index = (long) cookie;
+
+        if (!child_went_down (op_ret, op_errno) && (op_ret == -1)) {
+                local->op_ret   = -1;
+                local->op_errno = op_errno;
+
+                afr_lk_unlock (frame, this);
+                return 0;
+        }
+
+        if (op_ret == 0) {
+                local->op_ret        = 0;
+                local->op_errno      = 0;
+                local->cont.lk.locked_nodes[child_index] = 1;
+                local->cont.lk.ret_flock = *lock;
+        }
+
+        child_index++;
+
+        if (child_index < priv->child_count) {
+                STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) child_index,
+                                   priv->children[child_index],
+                                   priv->children[child_index]->fops->lk,
+                                   local->fd, local->cont.lk.cmd,
+                                   &local->cont.lk.user_flock, xdata);
+        } else if (local->op_ret == -1) {
+                /* all nodes have gone down */
+
+                AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
+                                  &local->cont.lk.ret_flock, NULL);
+        } else {
+                /* locking has succeeded on all nodes that are up */
+
+                /* temporarily
+                   ret = afr_mark_locked_nodes (this, local->fd,
+                   local->cont.lk.locked_nodes);
+                   if (ret)
+                   gf_log (this->name, GF_LOG_DEBUG,
+                   "Could not save locked nodes info in fdctx");
+
+                   ret = afr_save_locked_fd (this, local->fd);
+                   if (ret)
+                   gf_log (this->name, GF_LOG_DEBUG,
+                   "Could not save locked fd");
+
+                */
+                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
+                                  &local->cont.lk.ret_flock, NULL);
+        }
+
+        return 0;
+}
+
+
+int
+afr_lk (call_frame_t *frame, xlator_t *this,
+        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int i = 0;
+        int32_t op_errno = 0;
+        int     ret      = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->cont.lk.locked_nodes = GF_CALLOC (priv->child_count,
+                                                 sizeof (*local->cont.lk.locked_nodes),
+                                                 gf_afr_mt_char);
+
+        if (!local->cont.lk.locked_nodes) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        local->fd            = fd_ref (fd);
+        local->cont.lk.cmd   = cmd;
+        local->cont.lk.user_flock = *flock;
+        local->cont.lk.ret_flock = *flock;
+
+        STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
+                           priv->children[i],
+                           priv->children[i]->fops->lk,
+                           fd, cmd, flock, xdata);
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+afr_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t        ctx_addr = 0;
+        afr_inode_ctx_t *ctx     = NULL;
+
+        inode_ctx_get (inode, this, &ctx_addr);
+
+        if (!ctx_addr)
+                goto out;
+
+        ctx = (afr_inode_ctx_t *)(long)ctx_addr;
+        GF_FREE (ctx->fresh_children);
+        GF_FREE (ctx);
+out:
+        return 0;
+}
+
+int
+afr_priv_dump (xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        char  key_prefix[GF_DUMP_MAX_BUF_LEN];
+        char  key[GF_DUMP_MAX_BUF_LEN];
+        int   i = 0;
+
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+        snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+        gf_proc_dump_add_section(key_prefix);
+        gf_proc_dump_write("child_count", "%u", priv->child_count);
+        gf_proc_dump_write("read_child_rr", "%u", priv->read_child_rr);
+        for (i = 0; i < priv->child_count; i++) {
+                sprintf (key, "child_up[%d]", i);
+                gf_proc_dump_write(key, "%d", priv->child_up[i]);
+                sprintf (key, "pending_key[%d]", i);
+                gf_proc_dump_write(key, "%s", priv->pending_key[i]);
+        }
+        gf_proc_dump_write("data_self_heal", "%s", priv->data_self_heal);
+        gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
+        gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
+        gf_proc_dump_write("data_change_log", "%d", priv->data_change_log);
+        gf_proc_dump_write("metadata_change_log", "%d", priv->metadata_change_log);
+        gf_proc_dump_write("entry-change_log", "%d", priv->entry_change_log);
+        gf_proc_dump_write("read_child", "%d", priv->read_child);
+        gf_proc_dump_write("favorite_child", "%d", priv->favorite_child);
+        gf_proc_dump_write("wait_count", "%u", priv->wait_count);
+
+        return 0;
+}
+
+
+/**
+ * find_child_index - find the child's index in the array of subvolumes
+ * @this: AFR
+ * @child: child
+ */
+
+static int
+find_child_index (xlator_t *this, xlator_t *child)
+{
+        afr_private_t *priv = NULL;
+        int i = -1;
+
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((xlator_t *) child == priv->children[i])
+                        break;
+        }
+
+        return i;
+}
+
+int32_t
+afr_notify (xlator_t *this, int32_t event,
+            void *data, void *data2)
+{
+        afr_private_t   *priv               = NULL;
+        int             i                   = -1;
+        int             up_children         = 0;
+        int             down_children       = 0;
+        int             propagate           = 0;
+        int             had_heard_from_all  = 0;
+        int             have_heard_from_all = 0;
+        int             idx                 = -1;
+        int             ret                 = -1;
+        int             call_psh            = 0;
+        int             up_child            = AFR_ALL_CHILDREN;
+        dict_t          *input              = NULL;
+        dict_t          *output             = NULL;
+
+        priv = this->private;
+
+        if (!priv)
+                return 0;
+
+        /*
+         * We need to reset this in case children come up in "staggered"
+         * fashion, so that we discover a late-arriving local subvolume.  Note
+         * that we could end up issuing N lookups to the first subvolume, and
+         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+         */
+        priv->did_discovery = _gf_false;
+
+        had_heard_from_all = 1;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!priv->last_event[i]) {
+                        had_heard_from_all = 0;
+                }
+        }
+
+        /* parent xlators dont need to know about every child_up, child_down
+         * because of afr ha. If all subvolumes go down, child_down has
+         * to be triggered. In that state when 1 subvolume comes up child_up
+         * needs to be triggered. dht optimizes revalidate lookup by sending
+         * it only to one of its subvolumes. When child up/down happens
+         * for afr's subvolumes dht should be notified by child_modified. The
+         * subsequent revalidate lookup happens on all the dht's subvolumes
+         * which triggers afr self-heals if any.
+         */
+        idx = find_child_index (this, data);
+        if (idx < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Received child_up "
+                        "from invalid subvolume");
+                goto out;
+        }
+
+        switch (event) {
+        case GF_EVENT_CHILD_UP:
+                LOCK (&priv->lock);
+                {
+                        /*
+                         * This only really counts if the child was never up
+                         * (value = -1) or had been down (value = 0).  See
+                         * comment at GF_EVENT_CHILD_DOWN for a more detailed
+                         * explanation.
+                         */
+                        if (priv->child_up[idx] != 1) {
+                                priv->up_count++;
+                        }
+                        priv->child_up[idx] = 1;
+
+                        call_psh = 1;
+                        up_child = idx;
+                        for (i = 0; i < priv->child_count; i++)
+                                if (priv->child_up[i] == 1)
+                                        up_children++;
+                        if (up_children == 1) {
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "Subvolume '%s' came back up; "
+                                        "going online.", ((xlator_t *)data)->name);
+                        } else {
+                                event = GF_EVENT_CHILD_MODIFIED;
+                        }
+
+                        priv->last_event[idx] = event;
+                }
+                UNLOCK (&priv->lock);
+
+                break;
+
+        case GF_EVENT_CHILD_DOWN:
+                LOCK (&priv->lock);
+                {
+                        /*
+                         * If a brick is down when we start, we'll get a
+                         * CHILD_DOWN to indicate its initial state.  There
+                         * was never a CHILD_UP in this case, so if we
+                         * increment "down_count" the difference between than
+                         * and "up_count" will no longer be the number of
+                         * children that are currently up.  This has serious
+                         * implications e.g. for quorum enforcement, so we
+                         * don't increment these values unless the event
+                         * represents an actual state transition between "up"
+                         * (value = 1) and anything else.
+                         */
+                        if (priv->child_up[idx] == 1) {
+                                priv->down_count++;
+                        }
+                        priv->child_up[idx] = 0;
+
+                        for (i = 0; i < priv->child_count; i++)
+                                if (priv->child_up[i] == 0)
+                                        down_children++;
+                        if (down_children == priv->child_count) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "All subvolumes are down. Going offline "
+                                        "until atleast one of them comes back up.");
+                        } else {
+                                event = GF_EVENT_CHILD_MODIFIED;
+                        }
+
+                        priv->last_event[idx] = event;
+                }
+                UNLOCK (&priv->lock);
+
+                break;
+
+        case GF_EVENT_CHILD_CONNECTING:
+                LOCK (&priv->lock);
+                {
+                        priv->last_event[idx] = event;
+                }
+                UNLOCK (&priv->lock);
+
+                break;
+
+        case GF_EVENT_TRANSLATOR_OP:
+                input = data;
+                output = data2;
+                ret = afr_xl_op (this, input, output);
+                goto out;
+                break;
+
+        default:
+                propagate = 1;
+                break;
+        }
+
+        /* have all subvolumes reported status once by now? */
+        have_heard_from_all = 1;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!priv->last_event[i])
+                        have_heard_from_all = 0;
+        }
+
+        /* if all subvols have reported status, no need to hide anything
+           or wait for anything else. Just propagate blindly */
+        if (have_heard_from_all)
+                propagate = 1;
+
+        if (!had_heard_from_all && have_heard_from_all) {
+                /* This is the first event which completes aggregation
+                   of events from all subvolumes. If at least one subvol
+                   had come up, propagate CHILD_UP, but only this time
+                */
+                event = GF_EVENT_CHILD_DOWN;
+
+                LOCK (&priv->lock);
+                {
+                        up_children = afr_up_children_count (priv->child_up,
+                                                             priv->child_count);
+                        for (i = 0; i < priv->child_count; i++) {
+                                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
+                                        event = GF_EVENT_CHILD_UP;
+                                        break;
+                                }
+
+                                if (priv->last_event[i] ==
+                                                GF_EVENT_CHILD_CONNECTING) {
+                                        event = GF_EVENT_CHILD_CONNECTING;
+                                        /* continue to check other events for CHILD_UP */
+                                }
+                        }
+                }
+                UNLOCK (&priv->lock);
+        }
+
+        ret = 0;
+        if (propagate)
+                ret = default_notify (this, event, data);
+        if (call_psh && priv->shd.iamshd)
+                afr_proactive_self_heal ((void*) (long) up_child);
+
+out:
+        return ret;
+}
+
+int
+afr_first_up_child (unsigned char *child_up, size_t child_count)
+{
+        int         ret      = -1;
+        int         i        = 0;
+
+        GF_ASSERT (child_up);
+
+        for (i = 0; i < child_count; i++) {
+                if (child_up[i]) {
+                        ret = i;
+                        break;
+                }
+        }
+
+        return ret;
+}
+
+int
+afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
+{
+        int     ret = -1;
+
+        local->op_ret = -1;
+        local->op_errno = EUCLEAN;
+
+        local->child_up = GF_CALLOC (priv->child_count,
+                                     sizeof (*local->child_up),
+                                     gf_afr_mt_char);
+        if (!local->child_up) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+
+        memcpy (local->child_up, priv->child_up,
+                sizeof (*local->child_up) * priv->child_count);
+        local->call_count = afr_up_children_count (local->child_up,
+                                                   priv->child_count);
+        if (local->call_count == 0) {
+                gf_log (THIS->name, GF_LOG_INFO, "no subvolumes up");
+                if (op_errno)
+                        *op_errno = ENOTCONN;
+                goto out;
+        }
+
+        local->child_errno = GF_CALLOC (priv->child_count,
+                                        sizeof (*local->child_errno),
+                                        gf_afr_mt_int32_t);
+        if (!local->child_errno) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+
+        local->transaction.postop_piggybacked = GF_CALLOC (priv->child_count,
+							   sizeof (int),
+							   gf_afr_mt_int32_t);
+        if (!local->transaction.postop_piggybacked) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+
+	local->append_write = _gf_false;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
+                        transaction_lk_type_t lk_type)
+{
+        int             ret = -ENOMEM;
+
+        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
+                                      child_count, gf_afr_mt_char);
+        if (NULL == lk->locked_nodes)
+                goto out;
+
+        lk->lower_locked_nodes = GF_CALLOC (sizeof (*lk->lower_locked_nodes),
+                                            child_count, gf_afr_mt_char);
+        if (NULL == lk->lower_locked_nodes)
+                goto out;
+
+        lk->lock_op_ret   = -1;
+        lk->lock_op_errno = EUCLEAN;
+        lk->transaction_lk_type = lk_type;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+afr_matrix_cleanup (int32_t **matrix, unsigned int m)
+{
+        int             i         = 0;
+
+        if (!matrix)
+                goto out;
+        for (i = 0; i < m; i++) {
+                GF_FREE (matrix[i]);
+        }
+
+        GF_FREE (matrix);
+out:
+        return;
+}
+
+int32_t**
+afr_matrix_create (unsigned int m, unsigned int n)
+{
+        int32_t         **matrix = NULL;
+        int             i       = 0;
+
+        matrix = GF_CALLOC (sizeof (*matrix), m, gf_afr_mt_int32_t);
+        if (!matrix)
+                goto out;
+
+        for (i = 0; i < m; i++) {
+                matrix[i] = GF_CALLOC (sizeof (*matrix[i]), n,
+                                       gf_afr_mt_int32_t);
+                if (!matrix[i])
+                        goto out;
+        }
+        return matrix;
+out:
+        afr_matrix_cleanup (matrix, m);
+        return NULL;
+}
+
+int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count)
+{
+        int             ret = -ENOMEM;
+
+        lk->domain = dom;
+        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
+                                      child_count, gf_afr_mt_char);
+        if (NULL == lk->locked_nodes)
+                goto out;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+afr_transaction_local_init (afr_local_t *local, xlator_t *this)
+{
+        int            child_up_count = 0;
+        int            ret = -ENOMEM;
+        afr_private_t *priv = NULL;
+
+        priv = this->private;
+        ret = afr_internal_lock_init (&local->internal_lock, priv->child_count,
+                                      AFR_TRANSACTION_LK);
+        if (ret < 0)
+                goto out;
+
+        if ((local->transaction.type == AFR_DATA_TRANSACTION) ||
+            (local->transaction.type == AFR_METADATA_TRANSACTION)) {
+                ret = afr_inodelk_init (&local->internal_lock.inodelk[0],
+                                        this->name, priv->child_count);
+                if (ret < 0)
+                        goto out;
+        }
+
+        ret = -ENOMEM;
+        child_up_count = afr_up_children_count (local->child_up,
+                                                priv->child_count);
+        if (priv->optimistic_change_log && child_up_count == priv->child_count)
+                local->optimistic_change_log = 1;
+
+        local->first_up_child = afr_first_up_child (local->child_up,
+                                                    priv->child_count);
+
+        local->transaction.eager_lock =
+                GF_CALLOC (sizeof (*local->transaction.eager_lock),
+                           priv->child_count,
+                           gf_afr_mt_int32_t);
+
+        if (!local->transaction.eager_lock)
+                goto out;
+
+        local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children)
+                goto out;
+
+        local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
+                                               priv->child_count,
+                                               gf_afr_mt_char);
+        if (!local->transaction.pre_op)
+                goto out;
+
+        local->pending = afr_matrix_create (priv->child_count,
+                                            AFR_NUM_CHANGE_LOGS);
+        if (!local->pending)
+                goto out;
+
+        local->transaction.txn_changelog = afr_matrix_create (priv->child_count,
+                                                           AFR_NUM_CHANGE_LOGS);
+        if (!local->transaction.txn_changelog)
+                goto out;
+
+	INIT_LIST_HEAD (&local->transaction.eager_locked);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+afr_reset_children (int32_t *fresh_children, int32_t child_count)
+{
+        unsigned int i = 0;
+        for (i = 0; i < child_count; i++)
+                fresh_children[i] = -1;
+}
+
+int32_t*
+afr_children_create (int32_t child_count)
+{
+        int32_t           *children = NULL;
+        int               i               = 0;
+
+        GF_ASSERT (child_count > 0);
+
+        children = GF_CALLOC (child_count, sizeof (*children),
+                              gf_afr_mt_int32_t);
+        if (NULL == children)
+                goto out;
+        for (i = 0; i < child_count; i++)
+                children[i] = -1;
+out:
+        return children;
+}
+
+void
+afr_children_add_child (int32_t *children, int32_t child,
+                        int32_t child_count)
+{
+        gf_boolean_t child_found = _gf_false;
+        int          i               = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (children[i] == -1)
+                        break;
+                if (children[i] == child) {
+                        child_found = _gf_true;
+                        break;
+                }
+        }
+
+        if (!child_found) {
+                GF_ASSERT (i < child_count);
+                children[i] = child;
+        }
+}
+
+void
+afr_children_rm_child (int32_t *children, int32_t child, int32_t child_count)
+{
+        int          i = 0;
+
+        GF_ASSERT ((child >= 0) && (child < child_count));
+        for (i = 0; i < child_count; i++) {
+                if (children[i] == -1)
+                        break;
+                if (children[i] == child) {
+                        if (i != (child_count - 1))
+                                memmove (children + i, children + i + 1,
+                                         sizeof (*children)*(child_count - i - 1));
+                        children[child_count - 1] = -1;
+                        break;
+                }
+        }
+}
+
+int
+afr_get_children_count (int32_t *children, unsigned int child_count)
+{
+        int count = 0;
+        int i = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (children[i] == -1)
+                        break;
+                count++;
+        }
+        return count;
+}
+
+void
+afr_set_low_priority (call_frame_t *frame)
+{
+        frame->root->pid = LOW_PRIO_PROC_PID;
+}
+
+int
+afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
+                      int flags)
+{
+        int             ret = 0;
+        uint64_t        ctx = 0;
+        afr_fd_ctx_t    *fd_ctx      = NULL;
+
+        GF_ASSERT (fd && fd->inode);
+        ret = afr_fd_ctx_set (this, fd);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not set fd ctx for fd=%p", fd);
+                goto out;
+        }
+
+        ret = fd_ctx_get (fd, this, &ctx);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not get fd ctx for fd=%p", fd);
+                goto out;
+        }
+
+        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        fd_ctx->opened_on[child] = AFR_FD_OPENED;
+        if (!IA_ISDIR (fd->inode->ia_type)) {
+                fd_ctx->flags            = flags;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+gf_boolean_t
+afr_have_quorum (char *logname, afr_private_t *priv)
+{
+        unsigned int        quorum = 0;
+
+        GF_VALIDATE_OR_GOTO(logname,priv,out);
+
+        quorum = priv->quorum_count;
+        if (quorum != AFR_QUORUM_AUTO) {
+                return (priv->up_count >= (priv->down_count + quorum));
+        }
+
+        quorum = priv->child_count / 2 + 1;
+        if (priv->up_count >= (priv->down_count + quorum)) {
+                return _gf_true;
+        }
+
+        /*
+         * Special case for even numbers of nodes: if we have exactly half
+         * and that includes the first ("senior-most") node, then that counts
+         * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
+         * while preserving the critical property that there can only be one
+         * such group.
+         */
+        if ((priv->child_count % 2) == 0) {
+                quorum = priv->child_count / 2;
+                if (priv->up_count >= (priv->down_count + quorum)) {
+                        if (priv->child_up[0]) {
+                                return _gf_true;
+                        }
+                }
+        }
+
+out:
+        return _gf_false;
+}
+
+void
+afr_priv_destroy (afr_private_t *priv)
+{
+        int            i           = 0;
+
+        if (!priv)
+                goto out;
+        inode_unref (priv->root_inode);
+        GF_FREE (priv->shd.pos);
+        GF_FREE (priv->shd.pending);
+        GF_FREE (priv->shd.inprogress);
+//        for (i = 0; i < priv->child_count; i++)
+//                if (priv->shd.timer && priv->shd.timer[i])
+//                        gf_timer_call_cancel (this->ctx, priv->shd.timer[i]);
+        GF_FREE (priv->shd.timer);
+
+        if (priv->shd.healed)
+                eh_destroy (priv->shd.healed);
+
+        if (priv->shd.heal_failed)
+                eh_destroy (priv->shd.heal_failed);
+
+        if (priv->shd.split_brain)
+                eh_destroy (priv->shd.split_brain);
+
+        for (i = 0; i < priv->child_count; i++)
+        {
+                if (priv->shd.statistics[i])
+                        eh_destroy (priv->shd.statistics[i]);
+        }
+
+        GF_FREE (priv->shd.statistics);
+
+        GF_FREE (priv->shd.crawl_events);
+
+        GF_FREE (priv->last_event);
+        if (priv->pending_key) {
+                for (i = 0; i < priv->child_count; i++)
+                        GF_FREE (priv->pending_key[i]);
+        }
+        GF_FREE (priv->pending_key);
+        GF_FREE (priv->children);
+        GF_FREE (priv->child_up);
+        LOCK_DESTROY (&priv->lock);
+        LOCK_DESTROY (&priv->read_child_lock);
+        pthread_mutex_destroy (&priv->mutex);
+        GF_FREE (priv);
+out:
+        return;
+}
+
+int
+xlator_subvolume_count (xlator_t *this)
+{
+        int i = 0;
+        xlator_list_t *list = NULL;
+
+        for (list = this->children; list; list = list->next)
+                i++;
+        return i;
+}
+
+inline gf_boolean_t
+afr_is_errno_set (int *child_errno, int child)
+{
+        return child_errno[child];
+}
+
+inline gf_boolean_t
+afr_is_errno_unset (int *child_errno, int child)
+{
+        return !afr_is_errno_set (child_errno, child);
+}
+
+void
+afr_prepare_new_entry_pending_matrix (int32_t **pending,
+                                      gf_boolean_t (*is_pending) (int *, int),
+                                      int *ctx, struct iatt *buf,
+                                      unsigned int child_count)
+{
+        int midx = 0;
+        int idx  = 0;
+        int i    = 0;
+
+        midx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
+        if (IA_ISDIR (buf->ia_type))
+                idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+        else if (IA_ISREG (buf->ia_type))
+                idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+        else
+                idx = -1;
+        for (i = 0; i < child_count; i++) {
+                if (is_pending (ctx, i)) {
+                        pending[i][midx] = hton32 (1);
+                        if (idx == -1)
+                                continue;
+                        pending[i][idx] = hton32 (1);
+                }
+        }
+}
+
+gf_boolean_t
+afr_is_fd_fixable (fd_t *fd)
+{
+        if (!fd || !fd->inode)
+                return _gf_false;
+        else if (fd_is_anonymous (fd))
+                return _gf_false;
+        else if (uuid_is_null (fd->inode->gfid))
+                return _gf_false;
+
+        return _gf_true;
+}
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        inode_t         *inode = NULL;
+        afr_inode_ctx_t *ctx   = NULL;
+
+        local = frame->local;
+
+        if (local->fd)
+                inode = local->fd->inode;
+        else
+                inode = local->loc.inode;
+
+        if (!inode)
+                return;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __afr_inode_ctx_get (inode, this);
+                ctx->open_fd_count = local->open_fd_count;
+        }
+        UNLOCK (&inode->lock);
+}
+
+int
+afr_initialise_statistics (xlator_t *this)
+{
+        afr_private_t       *priv = NULL;
+        int                 ret = -1;
+        int                 i = 0;
+        int                 child_count = 0;
+        eh_t                *stats_per_brick = NULL;
+        shd_crawl_event_t   ***shd_crawl_events = NULL;
+        priv = this->private;
+
+        priv->shd.statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
+                                          gf_common_mt_eh_t);
+        if (!priv->shd.statistics) {
+                ret = -1;
+                goto out;
+        }
+        child_count = priv->child_count;
+        for (i=0; i < child_count ; i++) {
+                stats_per_brick = eh_new (AFR_STATISTICS_HISTORY_SIZE,
+                                          _gf_false,
+                                          _destroy_crawl_event_data);
+                if (!stats_per_brick) {
+                        ret = -1;
+                        goto out;
+                }
+                priv->shd.statistics[i] = stats_per_brick;
+
+        }
+
+        shd_crawl_events = (shd_crawl_event_t***)(&priv->shd.crawl_events);
+        *shd_crawl_events  = GF_CALLOC (sizeof(shd_crawl_event_t*),
+                                        priv->child_count,
+                                        gf_afr_mt_shd_crawl_event_t);
+
+        if (!priv->shd.crawl_events) {
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+
+}
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index 0c65ca852..689dd84e6 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -24,6 +15,7 @@
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
+#include <string.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -42,304 +34,512 @@
 #include "common-utils.h"
 #include "compat-errno.h"
 #include "compat.h"
+#include "checksum.h"
 
 #include "afr.h"
+#include "afr-self-heal.h"
+#include "afr-self-heal-common.h"
 
+int
+afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this, int32_t op_ret,
+                           int32_t op_errno, int32_t sh_failed)
+{
+        afr_local_t *local  = NULL;
 
-int32_t
-afr_opendir_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno,
-		 fd_t *fd)
+        local = frame->local;
+
+        afr_set_opendir_done (this, local->fd->inode);
+
+        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
+                          local->op_errno, local->fd, NULL);
+
+        return 0;
+}
+
+
+gf_boolean_t
+__checksums_differ (uint32_t *checksum, int child_count,
+                    unsigned char *child_up)
 {
-	afr_local_t * local  = NULL;
+        int          ret            = _gf_false;
+        int          i              = 0;
+        uint32_t     cksum          = 0;
+        gf_boolean_t activate_check = _gf_false;
+
+        for (i = 0; i < child_count; i++) {
+                if (!child_up[i])
+                        continue;
+                if (_gf_false == activate_check) {
+                        cksum          = checksum[i];
+                        activate_check = _gf_true;
+                        continue;
+                }
+
+                if (cksum != checksum[i]) {
+                        ret = _gf_true;
+                        break;
+                }
+
+                cksum = checksum[i];
+        }
+
+        return ret;
+}
 
-	int call_count = -1;
 
-	LOCK (&frame->lock);
-	{
-		local = frame->local;
+int32_t
+afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
+                             xlator_t *this, int32_t op_ret, int32_t op_errno,
+                             gf_dirent_t *entries, dict_t *xdata)
+{
+        afr_private_t *   priv        = NULL;
+        afr_local_t *     local       = NULL;
+        afr_self_heal_t * sh          = NULL;
+        gf_dirent_t *     entry       = NULL;
+        gf_dirent_t *     tmp         = NULL;
+        char              *reason     = NULL;
+        int               child_index = 0;
+        uint32_t          entry_cksum = 0;
+        int               call_count  = 0;
+        off_t             last_offset = 0;
+        inode_t           *inode      = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+        sh    = &local->self_heal;
+        inode = local->fd->inode;
+
+        child_index = (long) cookie;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: failed to do opendir on %s",
+                        local->loc.path, priv->children[child_index]->name);
+                local->op_ret = -1;
+                local->op_ret = op_errno;
+                goto out;
+        }
+
+        if (op_ret == 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: no entries found in %s",
+                        local->loc.path, priv->children[child_index]->name);
+                goto out;
+        }
+
+        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
+                entry_cksum = gf_rsync_weak_checksum ((unsigned char *)entry->d_name,
+                                                      strlen (entry->d_name));
+                local->cont.opendir.checksum[child_index] ^= entry_cksum;
+        }
+
+        list_for_each_entry (entry, &entries->list, list) {
+                last_offset = entry->d_off;
+        }
+
+        /* read more entries */
+
+        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->readdir,
+                           local->fd, 131072, last_offset, NULL);
+
+        return 0;
+
+out:
+        call_count = afr_frame_return (frame);
 
-		if (op_ret == 0)
-			local->op_ret = 0;
+        if (call_count == 0) {
+                if (__checksums_differ (local->cont.opendir.checksum,
+                                        priv->child_count,
+                                        local->child_up)) {
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+                        sh->do_entry_self_heal  = _gf_true;
+                        sh->forced_merge          = _gf_true;
 
-	call_count = afr_frame_return (frame);
+                        reason = "checksums of directory differ";
+                        afr_launch_self_heal (frame, this, inode, _gf_false,
+                                              inode->ia_type, reason, NULL,
+                                              afr_examine_dir_sh_unwind);
+                } else {
+                        afr_set_opendir_done (this, inode);
 
-	if (call_count == 0) {
-		AFR_STACK_UNWIND (frame, local->op_ret,
-				  local->op_errno, local->fd);
-	}
+                        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
+                                          local->op_errno, local->fd, NULL);
+                }
+        }
 
-	return 0;
+        return 0;
 }
 
 
-int32_t 
-afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd)
+int
+afr_examine_dir (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv        = NULL;
-	afr_local_t   * local       = NULL;
-
-	int             child_count = 0;
-	int             i           = 0;
+        afr_private_t * priv       = NULL;
+        afr_local_t *   local      = NULL;
+        int             i          = 0;
+        int             call_count = 0;
 
-	int ret = -1;
-	int call_count = -1;
+        local = frame->local;
+        priv  = this->private;
 
-	int32_t         op_ret   = -1;
-	int32_t         op_errno = 0;
+        local->cont.opendir.checksum = GF_CALLOC (priv->child_count,
+                                                  sizeof (*local->cont.opendir.checksum),
+                                                  gf_afr_mt_int32_t);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        call_count = afr_up_children_count (local->child_up, priv->child_count);
 
-	priv = this->private;
+        local->call_count = call_count;
 
-	child_count = priv->child_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->readdir,
+                                           local->fd, 131072, 0, NULL);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+                        if (!--call_count)
+                                break;
+                }
+        }
 
-	frame->local = local;
-	local->fd    = fd_ref (fd);
+        return 0;
+}
 
-	call_count = local->call_count;
-	
-	for (i = 0; i < child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_opendir_cbk, 
-				    priv->children[i],
-				    priv->children[i]->fops->opendir,
-				    loc, fd);
 
-			if (!--call_count)
-				break;
-		}
-	}
+int32_t
+afr_opendir_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno,
+                 fd_t *fd, dict_t *xdata)
+{
+        afr_private_t *priv              = NULL;
+        afr_local_t   *local             = NULL;
+        int32_t        up_children_count = 0;
+        int            ret               = -1;
+        int            call_count        = -1;
+        int32_t        child_index       = 0;
+
+        priv  = this->private;
+        local = frame->local;
+        child_index = (long) cookie;
+
+        up_children_count = afr_up_children_count (local->child_up,
+                                                   priv->child_count);
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret >= 0) {
+                        local->op_ret = op_ret;
+                        ret = afr_child_fd_ctx_set (this, fd, child_index, 0);
+                        if (ret) {
+                                local->op_ret = -1;
+                                local->op_errno = -ret;
+                                goto unlock;
+                        }
+                }
+
+                local->op_errno = op_errno;
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                if (local->op_ret != 0)
+                        goto out;
+
+                if (!afr_is_opendir_done (this, local->fd->inode) &&
+                    up_children_count > 1 && priv->entry_self_heal) {
+
+                        /*
+                         * This is the first opendir on this inode. We need
+                         * to check if the directory's entries are the same
+                         * on all subvolumes. This is needed in addition
+                         * to regular entry self-heal because the readdir
+                         * call is sent only to the first subvolume, and
+                         * thus files that exist only there will never be healed
+                         * otherwise (assuming changelog shows no anomalies).
+                         */
+
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "reading contents of directory %s looking for mismatch",
+                                local->loc.path);
+
+                        afr_examine_dir (frame, this);
+
+                } else {
+                        /* do the unwind */
+                        goto out;
+                }
+        }
+
+        return 0;
 
-	op_ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, fd);
-	}
+        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
+                          local->op_errno, local->fd, NULL);
 
-	return 0;
+        return 0;
 }
 
 
-/**
- * Common algorithm for directory read calls:
- * 
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: readdir
- */
-
 int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno,
-		 gf_dirent_t *buf)
+afr_opendir (call_frame_t *frame, xlator_t *this,
+             loc_t *loc, fd_t *fd)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+        afr_private_t * priv        = NULL;
+        afr_local_t   * local       = NULL;
+        int             child_count = 0;
+        int             i           = 0;
+        int             ret         = -1;
+        int             call_count  = -1;
+        int32_t         op_errno    = 0;
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	priv     = this->private;
-	children = priv->children;
+        priv = this->private;
 
-	local = frame->local;
+        child_count = priv->child_count;
 
-	if (op_ret == -1) {
-		last_tried = local->cont.readdir.last_tried;
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-		this_try = ++local->cont.readdir.last_tried;
-		unwind = 0;
+        loc_copy (&local->loc, loc);
 
-		STACK_WIND (frame, afr_readdir_cbk,
-			    children[this_try],
-			    children[this_try]->fops->readdir,
-			    local->fd, local->cont.readdir.size,
-			    local->cont.readdir.offset);
-	}
+        local->fd    = fd_ref (fd);
 
-out:
-	if (unwind) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, buf);
-	}
+        call_count = local->call_count;
 
-	return 0;
-}
+        for (i = 0; i < child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_opendir_cbk,
+                                           (void*) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->opendir,
+                                           loc, fd, NULL);
 
+                        if (!--call_count)
+                                break;
+                }
+        }
 
-int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset)
-{
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
-
-	int ret = -1;
-
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv     = this->private;
-	children = priv->children;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-						
-	frame->local = local;
-
-	call_child = afr_first_up_child (priv);
-	if (call_child == -1) {
-		op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR,
-			"no child is up :(");
-		goto out;
-	}
-
-	local->cont.readdir.last_tried = call_child;
-
-	local->fd                  = fd_ref (fd);
-	local->cont.readdir.size   = size;
-	local->cont.readdir.offset = offset;
-
-	STACK_WIND (frame, afr_readdir_cbk,
-		    children[call_child], children[call_child]->fops->readdir,
-		    fd, size, offset);
-
-	op_ret = 0;
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
-	return 0;
+        if (ret < 0)
+                AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
+
+        return 0;
 }
 
 
-int32_t
-afr_getdents_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dir_entry_t *entry, int32_t count)
-{
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+/**
+ * Common algorithm for directory read calls:
+ *
+ * - Try the fop on the first child that is up
+ * - if we have failed due to ENOTCONN:
+ *     try the next child
+ *
+ * Applicable to: readdir
+ */
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
 
-	priv     = this->private;
-	children = priv->children;
+struct entry_name {
+        char *name;
+        struct list_head list;
+};
 
-	local = frame->local;
+static void
+afr_forget_entries (fd_t *fd)
+{
+        struct entry_name *entry  = NULL;
+        struct entry_name *tmp    = NULL;
+        int                ret    = 0;
+        uint64_t           ctx    = 0;
+        afr_fd_ctx_t      *fd_ctx = NULL;
+
+        ret = fd_ctx_get (fd, THIS, &ctx);
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_INFO,
+                        "could not get fd ctx for fd=%p", fd);
+                return;
+        }
+
+        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+
+        list_for_each_entry_safe (entry, tmp, &fd_ctx->entries, list) {
+                GF_FREE (entry->name);
+                list_del (&entry->list);
+                GF_FREE (entry);
+        }
+}
 
-	if (op_ret == -1) {
-		last_tried = local->cont.getdents.last_tried;
+static void
+afr_readdir_filter_trash_dir (gf_dirent_t *entries, fd_t *fd)
+{
+        gf_dirent_t *   entry       = NULL;
+        gf_dirent_t *   tmp         = NULL;
+
+        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
+                if (__is_root_gfid (fd->inode->gfid) &&
+                    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
+                        list_del_init (&entry->list);
+                        GF_FREE (entry);
+                }
+        }
+}
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
+int32_t
+afr_readdir_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno,
+                 gf_dirent_t *entries, dict_t *xdata)
+{
+        afr_local_t     *local = NULL;
 
-		this_try = ++local->cont.getdents.last_tried;
-		unwind = 0;
+        if (op_ret == -1)
+                goto out;
 
-		STACK_WIND (frame, afr_getdents_cbk,
-			    children[this_try],
-			    children[this_try]->fops->getdents,
-			    local->fd, local->cont.getdents.size,
-			    local->cont.getdents.offset, local->cont.getdents.flag);
-	}
+        local = frame->local;
+        afr_readdir_filter_trash_dir (entries, local->fd);
 
 out:
-	if (unwind) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, entry, count);
-	}
-
-	return 0;
+        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, NULL);
+        return 0;
 }
 
 
 int32_t
-afr_getdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, size_t size, off_t offset, int32_t flag)
+afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
+        afr_local_t     *local = NULL;
 
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
+        if (op_ret == -1)
+                goto out;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        local = frame->local;
+        afr_readdir_filter_trash_dir (entries, local->fd);
 
-	priv     = this->private;
-	children = priv->children;
+out:
+        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, NULL);
+        return 0;
+}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+int32_t
+afr_do_readdir (call_frame_t *frame, xlator_t *this,
+                fd_t *fd, size_t size, off_t offset, int whichop, dict_t *dict)
+{
+        afr_private_t *priv      = NULL;
+        xlator_t      **children = NULL;
+        int           call_child = 0;
+        afr_local_t   *local     = NULL;
+        afr_fd_ctx_t  *fd_ctx    = NULL;
+        int           ret        = -1;
+        int32_t       op_errno   = 0;
+        uint64_t      read_child = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv     = this->private;
+        children = priv->children;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        read_child = afr_inode_get_read_ctx (this, fd->inode,
+                                             local->fresh_children);
+        ret = afr_get_call_child (this, local->child_up, read_child,
+                                  local->fresh_children,
+                                  &call_child,
+                                  &local->cont.readdir.last_index);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
+
+        fd_ctx  = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx) {
+                op_errno = EBADF;
+                goto out;
+        }
+
+        if ((offset == 0) || (fd_ctx->call_child == -1)) {
+                fd_ctx->call_child = call_child;
+        } else if ((priv->readdir_failover == _gf_false) &&
+                   (call_child != fd_ctx->call_child)) {
+                op_errno = EBADF;
+                goto out;
+        }
+
+        local->fd                  = fd_ref (fd);
+        local->cont.readdir.size   = size;
+        local->cont.readdir.dict   = (dict)? dict_ref (dict) : NULL;
+
+        if (whichop == GF_FOP_READDIR)
+                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
+                                   (void *) (long) call_child,
+                                   children[call_child],
+                                   children[call_child]->fops->readdir, fd,
+                                   size, offset, dict);
+        else
+                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
+                                   (void *) (long) call_child,
+                                   children[call_child],
+                                   children[call_child]->fops->readdirp, fd,
+                                   size, offset, dict);
+
+        return 0;
+out:
+        AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
 
-	call_child = afr_first_up_child (priv);
-	if (call_child == -1) {
-		op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR,
-			"no child is up :(");
-		goto out;
-	}
 
-	local->cont.getdents.last_tried = call_child;
+int32_t
+afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *xdata)
+{
+        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+        return 0;
+}
 
-	local->fd                   = fd_ref (fd);
 
-	local->cont.getdents.size   = size;
-	local->cont.getdents.offset = offset;
-	local->cont.getdents.flag   = flag;
-	
-	frame->local = local;
+int32_t
+afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *dict)
+{
+        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
+        return 0;
+}
 
-	STACK_WIND (frame, afr_getdents_cbk,
-		    children[call_child], children[call_child]->fops->getdents,
-		    fd, size, offset, flag);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+int32_t
+afr_releasedir (xlator_t *this, fd_t *fd)
+{
+        afr_forget_entries (fd);
+        afr_cleanup_fd_ctx (this, fd);
 
-	return 0;
+        return 0;
 }
-
-
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index 172ec3c90..09456d159 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_READ_H__
@@ -23,25 +14,23 @@
 
 int32_t
 afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd);
+	     loc_t *loc, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_closedir (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd);
+afr_releasedir (xlator_t *this, fd_t *fd);
 
 int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset);
+	     fd_t *fd, size_t size, off_t offset, dict_t *xdata);
 
 
 int32_t
-afr_getdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, size_t size, off_t offset, int32_t flag);
-
+afr_readdirp (call_frame_t *frame, xlator_t *this,
+              fd_t *fd, size_t size, off_t offset, dict_t *dict);
 
 int32_t
 afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags);
+	      loc_t *loc, int32_t flags, dict_t *xdata);
 
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 87a6e09b5..1943b719b 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -47,230 +38,470 @@
 #include "afr.h"
 #include "afr-transaction.h"
 
-
-void
-afr_build_parent_loc (loc_t *parent, loc_t *child)
+int
+afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
 {
-	char *tmp = NULL;
-
-	if (!child->parent) {
-		loc_copy (parent, child);
-		return;
-	}
-
-	tmp = strdup (child->path);
-	parent->path   = strdup (dirname (tmp));
-	FREE (tmp);
-
-        parent->name   = strrchr (parent->path, '/');
-	if (parent->name)
-		parent->name++;
+        int     ret = -1;
+        char    *child_path = NULL;
+
+        if (!child->parent) {
+                if (op_errno)
+                        *op_errno = EINVAL;
+                goto out;
+        }
+
+        child_path = gf_strdup (child->path);
+        if (!child_path) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+        parent->path = gf_strdup( dirname (child_path) );
+	if (!parent->path) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+        parent->inode  = inode_ref (child->parent);
+        uuid_copy (parent->gfid, child->pargfid);
+
+        ret = 0;
+out:
+	GF_FREE(child_path);
 
-	parent->inode  = inode_ref (child->parent);
-	parent->parent = inode_parent (parent->inode, 0, NULL);
-	parent->ino    = parent->inode->ino;
+        return ret;
 }
 
-
-/* {{{ create */
-
-int
-afr_create_unwind (call_frame_t *frame, xlator_t *this)
+void
+__dir_entry_fop_common_cbk (call_frame_t *frame, int child_index,
+                            xlator_t *this, int32_t op_ret,
+                            int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, struct iatt *preparent,
+                            struct iatt *postparent, struct iatt *prenewparent,
+                            struct iatt *postnewparent)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame)
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  local->cont.create.fd,
-				  local->cont.create.inode,
-				  &local->cont.create.buf);
-	return 0;
+        afr_local_t     *local          = NULL;
+
+        local = frame->local;
+
+        if (afr_fop_failed (op_ret, op_errno))
+                afr_transaction_fop_failed (frame, this, child_index);
+
+        if (op_ret > -1) {
+                local->op_ret = op_ret;
+
+                if ((local->success_count == 0) ||
+                    (child_index == local->read_child_index)) {
+                        local->cont.dir_fop.preparent      = *preparent;
+                        local->cont.dir_fop.postparent     = *postparent;
+                        if (buf)
+                                local->cont.dir_fop.buf            = *buf;
+                        if (prenewparent)
+                             local->cont.dir_fop.prenewparent  = *prenewparent;
+                        if (postnewparent)
+                             local->cont.dir_fop.postnewparent = *postnewparent;
+                }
+
+                local->cont.dir_fop.inode = inode;
+
+                local->fresh_children[local->success_count] = child_index;
+                local->success_count++;
+                local->child_errno[child_index] = 0;
+        } else {
+                local->child_errno[child_index] = op_errno;
+        }
+
+        local->op_errno = op_errno;
 }
 
-
 int
-afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno, 
-		     fd_t *fd, inode_t *inode, struct stat *buf)
+afr_mark_new_entry_changelog_cbk (call_frame_t *frame, void *cookie,
+                                  xlator_t *this,
+                                  int32_t op_ret, int32_t op_errno,
+                                  dict_t *xattr, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        int     call_count = 0;
 
-	int call_count = -1;
-	int child_index = -1;
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+                AFR_STACK_DESTROY (frame);
+        }
+        return 0;
+}
 
-	local = frame->local;
-	priv = this->private;
+void
+afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t  *new_frame  = NULL;
+        afr_local_t   *local      = NULL;
+        afr_local_t   *new_local  = NULL;
+        afr_private_t *priv       = NULL;
+        dict_t        **xattr     = NULL;
+        int32_t       **changelog = NULL;
+        int           i           = 0;
+        GF_UNUSED int op_errno    = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame) {
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (new_frame->local, out);
+        new_local = new_frame->local;
+        changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog)
+                goto out;
+
+        xattr = GF_CALLOC (priv->child_count, sizeof (*xattr),
+                           gf_afr_mt_dict_t);
+        if (!xattr)
+                goto out;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_errno[i])
+                        continue;
+                xattr[i] = dict_new ();
+                if (!xattr[i])
+                        goto out;
+        }
+
+        afr_prepare_new_entry_pending_matrix (changelog,
+                                              afr_is_errno_set,
+                                              local->child_errno,
+                                              &local->cont.dir_fop.buf,
+                                              priv->child_count);
+
+        new_local->pending = changelog;
+        uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+        new_local->loc.inode = inode_ref (local->cont.dir_fop.inode);
+        new_local->call_count = local->success_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_errno[i])
+                        continue;
+
+                afr_set_pending_dict (priv, xattr[i], changelog, i, LOCAL_LAST);
+                STACK_WIND_COOKIE (new_frame, afr_mark_new_entry_changelog_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->xattrop,
+                                   &new_local->loc, GF_XATTROP_ADD_ARRAY,
+                                   xattr[i], NULL);
+        }
+        new_frame = NULL;
+out:
+        if (new_frame)
+                AFR_STACK_DESTROY (new_frame);
+        afr_xattr_array_destroy (xattr, priv->child_count);
+        return;
+}
 
-	child_index = (long) cookie;
+gf_boolean_t
+afr_is_new_entry_changelog_needed (glusterfs_fop_t fop)
+{
+        glusterfs_fop_t fops[]   = {GF_FOP_CREATE, GF_FOP_MKNOD, GF_FOP_NULL};
+        int             i        = 0;
+
+        for (i = 0; fops[i] != GF_FOP_NULL; i++) {
+                if (fop == fops[i])
+                        return _gf_true;
+        }
+        return _gf_false;
+}
 
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
+void
+afr_dir_fop_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
 
-		if (op_ret != -1) {
-			local->op_ret = op_ret;
+        local = frame->local;
+        priv  = this->private;
 
-			if ((local->success_count == 0)
-			    || (child_index == priv->read_child)) {
-				local->cont.create.buf        = *buf;
-				local->cont.create.buf.st_ino = 
-					afr_itransform (buf->st_ino,
-							priv->child_count,
-							child_index);
-			}
-			local->cont.create.inode = inode;
+        if (local->op_ret < 0)
+                goto out;
 
-			local->success_count++;
-		}
+        if (local->success_count == priv->child_count)
+                goto out;
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        if (!afr_is_new_entry_changelog_needed (local->op))
+                goto out;
 
-	call_count = afr_frame_return (frame);
+        afr_mark_new_entry_changelog (frame, this);
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
+out:
+        return;
+}
 
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+void
+afr_dir_fop_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local          = NULL;
+        afr_private_t   *priv           = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+
+        if (local->cont.dir_fop.inode == NULL)
+                goto done;
+        afr_set_read_ctx_from_policy (this, local->cont.dir_fop.inode,
+                                      local->fresh_children,
+                                      local->read_child_index,
+                                      priv->read_child,
+                                      local->cont.dir_fop.buf.ia_gfid);
+done:
+        local->transaction.unwind (frame, this);
+        afr_dir_fop_mark_entry_pending_changelog (frame, this);
+        local->transaction.resume (frame, this);
 }
 
+/* {{{ create */
 
 int
-afr_create_wind (call_frame_t *frame, xlator_t *this)
+afr_create_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_create_wind_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->create,
-					   &local->loc, 
-					   local->cont.create.flags, 
-					   local->cont.create.mode, 
-					   local->cont.create.fd);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (create, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  local->cont.create.fd,
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  local->xdata_rsp);
+        }
+
+        return 0;
 }
 
 
 int
-afr_create_done (call_frame_t *frame, xlator_t *this)
+afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     fd_t *fd, inode_t *inode, struct iatt *buf,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
-	afr_local_t * local = NULL;
+        afr_local_t     *local = NULL;
+        uint64_t        ctx = 0;
+        afr_fd_ctx_t    *fd_ctx = NULL;
+        int             ret = 0;
+        int             call_count = -1;
+        int             child_index = -1;
+
+        local = frame->local;
+
+        child_index = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret > -1) {
+                        ret = afr_fd_ctx_set (this, fd);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not set ctx on fd=%p", fd);
+
+                                local->op_ret   = -1;
+                                local->op_errno = -ret;
+                                goto unlock;
+                        }
+
+                        ret = fd_ctx_get (fd, this, &ctx);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not get fd ctx for fd=%p", fd);
+                                local->op_ret   = -1;
+                                local->op_errno = -ret;
+                                goto unlock;
+                        }
+
+                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+
+                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+                        fd_ctx->flags                  = local->cont.create.flags;
+
+                        if (local->success_count == 0) {
+				if (xdata)
+					local->xdata_rsp = dict_ref(xdata);
+			}
+                }
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
+        }
 
-	local = frame->local;
+unlock:
+        UNLOCK (&frame->lock);
 
-	local->transaction.unwind (frame, this);
+        call_count = afr_frame_return (frame);
 
-	AFR_STACK_DESTROY (frame);
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
-	return 0;
+        return 0;
 }
 
 
 int
-afr_create (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t flags, mode_t mode, fd_t *fd)
+afr_create_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_create_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->create,
+                                           &local->loc,
+                                           local->cont.create.flags,
+                                           local->cont.create.mode,
+                                           local->umask,
+                                           local->cont.create.fd,
+                                           local->xdata_req);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
 
-	transaction_frame->local = local;
 
-	loc_copy (&local->loc, loc);
+int
+afr_create_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t * local = NULL;
 
-	local->cont.create.flags = flags;
-	local->cont.create.mode  = mode;
-	local->cont.create.fd    = fd_ref (fd);
+        local = frame->local;
 
-	local->transaction.fop    = afr_create_wind;
-	local->transaction.done   = afr_create_done;
-	local->transaction.unwind = afr_create_unwind;
+        local->transaction.unwind (frame, this);
 
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
+        AFR_STACK_DESTROY (frame);
 
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-	local->transaction.pending  = AFR_ENTRY_PENDING;
+        return 0;
+}
 
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
 
-	op_ret = 0;
+int
+afr_create (call_frame_t *frame, xlator_t *this,
+            loc_t *loc, int32_t flags, mode_t mode,
+            mode_t umask, fd_t *fd, dict_t *params)
+{
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        QUORUM_CHECK(create,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        loc_copy (&local->loc, loc);
+
+        LOCK (&priv->read_child_lock);
+        {
+                local->read_child_index = (++priv->read_child_rr)
+                        % (priv->child_count);
+        }
+        UNLOCK (&priv->read_child_lock);
+
+        local->op                = GF_FOP_CREATE;
+        local->cont.create.flags = flags;
+        local->cont.create.mode  = mode;
+        local->cont.create.fd    = fd_ref (fd);
+        local->umask  = umask;
+        if (params)
+                local->xdata_req = dict_ref (params);
+
+        local->transaction.fop    = afr_create_wind;
+        local->transaction.done   = afr_create_done;
+        local->transaction.unwind = afr_create_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
-
-	return 0;
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (create, frame, -1, op_errno,
+                                  NULL, NULL, NULL, NULL, NULL, NULL);
+        }
+
+        return 0;
 }
 
 /* }}} */
@@ -280,194 +511,201 @@ out:
 int
 afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame)
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  local->cont.mknod.inode,
-				  &local->cont.mknod.buf);
-	return 0;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (mknod, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
+        }
+
+        return 0;
 }
 
 
 int
-afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		    int32_t op_ret, int32_t op_errno, 
-		    inode_t *inode, struct stat *buf)
+afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
-		
-		if (op_ret != -1) {
-			local->op_ret = op_ret;
-
-			if ((local->success_count == 0)
-			    || (child_index == priv->read_child)) {	
-				local->cont.mknod.buf   = *buf;
-				local->cont.mknod.buf.st_ino = 
-					afr_itransform (buf->st_ino,
-							priv->child_count,
-							child_index);
-			}
-			local->cont.mknod.inode = inode;
+        int             call_count      = -1;
+        int             child_index     = -1;
 
-			local->success_count++;
-		}
+        child_index = (long) cookie;
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        LOCK (&frame->lock);
+        {
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
+        }
+        UNLOCK (&frame->lock);
 
-	call_count = afr_frame_return (frame);
+        call_count = afr_frame_return (frame);
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return 0;
 }
 
 
 int32_t
 afr_mknod_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->mknod,
-					   &local->loc, local->cont.mknod.mode,
-					   local->cont.mknod.dev);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->mknod,
+                                           &local->loc, local->cont.mknod.mode,
+                                           local->cont.mknod.dev,
+                                           local->umask,
+                                           local->xdata_req);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int
 afr_mknod_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = NULL;
+        afr_local_t * local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	local->transaction.unwind (frame, this);
-	AFR_STACK_DESTROY (frame);
+        local->transaction.unwind (frame, this);
+        AFR_STACK_DESTROY (frame);
 
-	return 0;
+        return 0;
 }
 
 
 int
-afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev)
+afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t dev, mode_t umask, dict_t *params)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc, loc);
-
-	local->cont.mknod.mode  = mode;
-	local->cont.mknod.dev   = dev;
-
-	local->transaction.fop    = afr_mknod_wind;
-	local->transaction.done   = afr_mknod_done;
-	local->transaction.unwind = afr_mknod_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-	local->transaction.pending  = AFR_ENTRY_PENDING;
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        QUORUM_CHECK(mknod,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        loc_copy (&local->loc, loc);
+
+        LOCK (&priv->read_child_lock);
+        {
+                local->read_child_index = (++priv->read_child_rr)
+                        % (priv->child_count);
+        }
+        UNLOCK (&priv->read_child_lock);
+
+        local->op               = GF_FOP_MKNOD;
+        local->cont.mknod.mode  = mode;
+        local->cont.mknod.dev   = dev;
+        local->umask = umask;
+        if (params)
+                local->xdata_req = dict_ref (params);
+
+        local->transaction.fop    = afr_mknod_wind;
+        local->transaction.done   = afr_mknod_done;
+        local->transaction.unwind = afr_mknod_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
-
-	return 0;
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (mknod, frame, -1, op_errno,
+                                  NULL, NULL, NULL, NULL, NULL);
+        }
+
+        return 0;
 }
 
 /* }}} */
@@ -478,194 +716,201 @@ out:
 int
 afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame)
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  local->cont.mkdir.inode,
-				  &local->cont.mkdir.buf);
-	return 0;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (mkdir, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
+        }
+
+        return 0;
 }
 
 
 int
-afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		    int32_t op_ret, int32_t op_errno, 
-		    inode_t *inode, struct stat *buf)
+afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
-
-		if (op_ret != -1) {
-			local->op_ret           = op_ret;
-
-			if ((local->success_count == 0)
-			    || (child_index == priv->read_child)) {
-				local->cont.mkdir.buf   = *buf;
-				local->cont.mkdir.buf.st_ino = 
-					afr_itransform (buf->st_ino, priv->child_count,
-							child_index);
-			}
-			local->cont.mkdir.inode = inode;
+        int             call_count      = -1;
+        int             child_index     = -1;
 
-			local->success_count++;
-		}
+        child_index = (long) cookie;
 
-		local->op_errno         = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        LOCK (&frame->lock);
+        {
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
+        }
+        UNLOCK (&frame->lock);
 
-	call_count = afr_frame_return (frame);
+        call_count = afr_frame_return (frame);
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return 0;
 }
 
 
 int
 afr_mkdir_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->mkdir,
-					   &local->loc, local->cont.mkdir.mode);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->mkdir,
+                                           &local->loc, local->cont.mkdir.mode,
+                                           local->umask,
+                                           local->xdata_req);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int
 afr_mkdir_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = NULL;
+        afr_local_t * local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	local->transaction.unwind (frame, this);
+        local->transaction.unwind (frame, this);
 
-	AFR_STACK_DESTROY (frame);
+        AFR_STACK_DESTROY (frame);
 
-	return 0;
+        return 0;
 }
 
-
 int
 afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode)
+           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc, loc);
-
-	local->cont.mkdir.mode  = mode;
-
-	local->transaction.fop    = afr_mkdir_wind;
-	local->transaction.done   = afr_mkdir_done;
-	local->transaction.unwind = afr_mkdir_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-	local->transaction.pending  = AFR_ENTRY_PENDING;
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        QUORUM_CHECK(mkdir,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        loc_copy (&local->loc, loc);
+
+        LOCK (&priv->read_child_lock);
+        {
+                local->read_child_index = (++priv->read_child_rr)
+                        % (priv->child_count);
+        }
+        UNLOCK (&priv->read_child_lock);
+
+        local->cont.mkdir.mode  = mode;
+        local->umask = umask;
+        if (params)
+                local->xdata_req = dict_ref (params);
+
+        local->op = GF_FOP_MKDIR;
+        local->transaction.fop    = afr_mkdir_wind;
+        local->transaction.done   = afr_mkdir_done;
+        local->transaction.unwind = afr_mkdir_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
 
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+                AFR_STACK_UNWIND (mkdir, frame, -1, op_errno,
+                                  NULL, NULL, NULL, NULL, NULL);
+        }
 
-	return 0;
+        return 0;
 }
 
 /* }}} */
@@ -676,198 +921,196 @@ out:
 int
 afr_link_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		local->cont.link.buf.st_ino = local->cont.link.ino;
-
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno, 
-				  local->cont.link.inode,
-				  &local->cont.link.buf);
-	}
-
-	return 0;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (link, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
+        }
+
+        return 0;
 }
 
 
 int
-afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		   int32_t op_ret, int32_t op_errno, inode_t *inode,
-		   struct stat *buf)
+afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        int             call_count      = -1;
+        int             child_index     = -1;
 
-	int call_count = -1;
-	int child_index = -1;
+        child_index = (long) cookie;
 
-	local = frame->local;
-	priv = this->private;
+        LOCK (&frame->lock);
+        {
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
+        }
+        UNLOCK (&frame->lock);
 
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
-
-		if (op_ret != -1) {
-			local->op_ret   = op_ret;
-
-			if ((local->success_count == 0)
-			    || (child_index == priv->read_child)) {
-				local->cont.link.buf        = *buf;
-				local->cont.link.buf.st_ino = 
-					afr_itransform (buf->st_ino, priv->child_count,
-							child_index);
-			}
-			local->cont.link.inode    = inode;
+        call_count = afr_frame_return (frame);
 
-			local->success_count++;
-		}
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
-		local->op_errno = op_errno;		
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return 0;
 }
 
 
 int
 afr_link_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->link,
-					   &local->loc,
-					   &local->newloc);
-			
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_link_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->link,
+                                           &local->loc,
+                                           &local->newloc, local->xdata_req);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int
 afr_link_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = frame->local;
+        afr_local_t * local = frame->local;
 
-	local->transaction.unwind (frame, this);
+        local->transaction.unwind (frame, this);
 
-	AFR_STACK_DESTROY (frame);
+        AFR_STACK_DESTROY (frame);
 
-	return 0;
+        return 0;
 }
 
 
 int
 afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc)
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc,    oldloc);
-	loc_copy (&local->newloc, newloc);
-
-	local->cont.link.ino = oldloc->inode->ino;
-
-	local->transaction.fop    = afr_link_wind;
-	local->transaction.done   = afr_link_done;
-	local->transaction.unwind = afr_link_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, oldloc);
-
-	local->transaction.main_frame   = frame;
-	local->transaction.basename     = AFR_BASENAME (oldloc->path);
-	local->transaction.new_basename = AFR_BASENAME (newloc->path);
-	local->transaction.pending      = AFR_ENTRY_PENDING;
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        QUORUM_CHECK(link,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        loc_copy (&local->loc,    oldloc);
+        loc_copy (&local->newloc, newloc);
+        if (xdata)
+                local->xdata_req = dict_ref (xdata);
+
+        LOCK (&priv->read_child_lock);
+        {
+                local->read_child_index = (++priv->read_child_rr)
+                        % (priv->child_count);
+        }
+        UNLOCK (&priv->read_child_lock);
+
+        local->op = GF_FOP_LINK;
+        local->transaction.fop    = afr_link_wind;
+        local->transaction.done   = afr_link_done;
+        local->transaction.unwind = afr_link_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, newloc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame   = frame;
+        local->transaction.basename     = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-
-	return 0;
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (link, frame, -1, op_errno,
+                                  NULL, NULL, NULL, NULL, NULL);
+        }
+
+        return 0;
 }
 
 /* }}} */
@@ -878,195 +1121,202 @@ out:
 int
 afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame)
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  local->cont.symlink.inode,
-				  &local->cont.symlink.buf);
-	return 0;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (symlink, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  local->cont.dir_fop.inode,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
+        }
+
+        return 0;
 }
 
 
 int
-afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		      int32_t op_ret, int32_t op_errno, inode_t *inode,
-		      struct stat *buf)
+afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
-		
-		if (op_ret != -1) {
-			local->op_ret   = op_ret;
-
-			if ((local->success_count == 0)
-			    || (child_index == priv->read_child)) {
-				local->cont.symlink.buf        = *buf;
-				local->cont.symlink.buf.st_ino = 
-					afr_itransform (buf->st_ino, priv->child_count,
-							child_index);
-			}
-			local->cont.symlink.inode    = inode;
+        int             call_count      = -1;
+        int             child_index     = -1;
 
-			local->success_count++;
-		}
+        child_index = (long) cookie;
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        LOCK (&frame->lock);
+        {
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, inode, buf,
+                                            preparent, postparent, NULL, NULL);
+        }
+        UNLOCK (&frame->lock);
 
-	call_count = afr_frame_return (frame);
+        call_count = afr_frame_return (frame);
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return 0;
 }
 
 
 int
 afr_symlink_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->symlink,
-					   local->cont.symlink.linkpath,
-					   &local->loc);
-
-			if (!--call_count)
-				break;
-
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->symlink,
+                                           local->cont.symlink.linkpath,
+                                           &local->loc,
+                                           local->umask,
+                                           local->xdata_req);
+
+                        if (!--call_count)
+                                break;
+
+                }
+        }
+
+        return 0;
 }
 
 
 int
 afr_symlink_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = frame->local;
+        afr_local_t * local = frame->local;
+
+        local->transaction.unwind (frame, this);
 
-	local->transaction.unwind (frame, this);
+        AFR_STACK_DESTROY (frame);
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+        return 0;
 }
 
 
 int
 afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *loc)
+             const char *linkpath, loc_t *loc, mode_t umask, dict_t *params)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-	
-	loc_copy (&local->loc, loc);
-
-	local->cont.symlink.ino      = loc->inode->ino;
-	local->cont.symlink.linkpath = strdup (linkpath);
-
-	local->transaction.fop    = afr_symlink_wind;
-	local->transaction.done   = afr_symlink_done;
-	local->transaction.unwind = afr_symlink_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame   = frame;
-	local->transaction.basename     = AFR_BASENAME (loc->path);
-	local->transaction.pending      = AFR_ENTRY_PENDING;
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        QUORUM_CHECK(symlink,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        loc_copy (&local->loc, loc);
+
+        LOCK (&priv->read_child_lock);
+        {
+                local->read_child_index = (++priv->read_child_rr)
+                        % (priv->child_count);
+        }
+        UNLOCK (&priv->read_child_lock);
+
+        local->cont.symlink.linkpath = gf_strdup (linkpath);
+        local->umask = umask;
+        if (params)
+                local->xdata_req = dict_ref (params);
+
+        local->op = GF_FOP_SYMLINK;
+        local->transaction.fop    = afr_symlink_wind;
+        local->transaction.done   = afr_symlink_done;
+        local->transaction.unwind = afr_symlink_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame   = frame;
+        local->transaction.basename     = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL, NULL);
-	}
-
-	return 0;
+        if (ret  < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (symlink, frame, -1, op_errno,
+                                  NULL, NULL, NULL, NULL, NULL);
+        }
+
+        return 0;
 }
 
 /* }}} */
@@ -1076,195 +1326,231 @@ out:
 int
 afr_rename_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (rename, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  &local->cont.dir_fop.buf,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  &local->cont.dir_fop.prenewparent,
+                                  &local->cont.dir_fop.postnewparent,
+                                  NULL);
+        }
+
+        return 0;
+}
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
 
-	if (main_frame) {
-		local->cont.rename.buf.st_ino = local->cont.rename.ino;
+int
+afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        int call_count = -1;
+        int child_index = -1;
 
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno, 
-				  &local->cont.rename.buf);
-	}
+        local = frame->local;
 
-	return 0;
-}
+        child_index = (long) cookie;
 
+        LOCK (&frame->lock);
+        {
+                if (afr_fop_failed (op_ret, op_errno) && op_errno != ENOTEMPTY)
+                        afr_transaction_fop_failed (frame, this, child_index);
+                local->op_errno = op_errno;
+                local->child_errno[child_index] = op_errno;
 
-int
-afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno, struct stat *buf)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv  = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
-
-		if ((op_ret != -1) && (local->success_count == 0)) {
-			local->op_ret = op_ret;
-		
-			if (buf) {
-				local->cont.rename.buf = *buf;
-				local->cont.rename.buf.st_ino = 
-					afr_itransform (buf->st_ino, priv->child_count,
-							child_index);
-			}
-			local->success_count++;
-		}
+                if (op_ret > -1)
+                        __dir_entry_fop_common_cbk (frame, child_index, this,
+                                                   op_ret, op_errno, NULL, buf,
+                                                   preoldparent, postoldparent,
+                                                   prenewparent, postnewparent);
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        }
+        UNLOCK (&frame->lock);
 
-	call_count = afr_frame_return (frame);
+        call_count = afr_frame_return (frame);
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
 
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return 0;
 }
 
 
 int32_t
 afr_rename_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_rename_wind_cbk, 
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->rename,
-					   &local->loc,
-					   &local->newloc);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_rename_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->rename,
+                                           &local->loc,
+                                           &local->newloc, NULL);
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int
 afr_rename_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = frame->local;
+        afr_local_t * local = frame->local;
+
+        local->transaction.unwind (frame, this);
 
-	local->transaction.unwind (frame, this);
+        AFR_STACK_DESTROY (frame);
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+        return 0;
 }
 
 
 int
 afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc)
+            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc,    oldloc);
-	loc_copy (&local->newloc, newloc);
-
-	local->cont.rename.ino = oldloc->inode->ino;
-
-	local->transaction.fop    = afr_rename_wind;
-	local->transaction.done   = afr_rename_done;
-	local->transaction.unwind = afr_rename_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, oldloc);
-	afr_build_parent_loc (&local->transaction.new_parent_loc, newloc);
-
-	local->transaction.main_frame   = frame;
-	local->transaction.basename     = AFR_BASENAME (oldloc->path);
-	local->transaction.new_basename = AFR_BASENAME (newloc->path);
-	local->transaction.pending      = AFR_ENTRY_PENDING;
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
-
-	op_ret = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+        int                     nlockee                 = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        QUORUM_CHECK(rename,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        loc_copy (&local->loc,    oldloc);
+        loc_copy (&local->newloc, newloc);
+
+        local->read_child_index = afr_inode_get_read_ctx (this, oldloc->inode, NULL);
+
+        local->op = GF_FOP_RENAME;
+        local->transaction.fop    = afr_rename_wind;
+        local->transaction.done   = afr_rename_done;
+        local->transaction.unwind = afr_rename_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, oldloc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+        ret = afr_build_parent_loc (&local->transaction.new_parent_loc, newloc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame   = frame;
+        local->transaction.basename     = AFR_BASENAME (oldloc->path);
+        local->transaction.new_basename = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.new_parent_loc,
+                                     local->transaction.new_basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        if (local->newloc.inode && IA_ISDIR (local->newloc.inode->ia_type)) {
+                ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                             &local->newloc,
+                                             NULL,
+                                             priv->child_count);
+                if (ret)
+                        goto out;
+
+                nlockee++;
+        }
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
+
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
 
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+                AFR_STACK_UNWIND (rename, frame, -1, op_errno,
+                                  NULL, NULL, NULL, NULL, NULL, NULL);
+        }
 
-	return 0;
+        return 0;
 }
 
 /* }}} */
@@ -1274,183 +1560,190 @@ out:
 int
 afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame)
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno);
-
-	return 0;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (unlink, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
+        }
+
+        return 0;
 }
 
 
 int
-afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno)
+afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-	int need_unwind = 0;
-
-	local = frame->local;
-	priv  = this->private;
-	
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
-
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret   = op_ret;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        afr_local_t *   local = NULL;
+        int call_count  = -1;
+        int child_index = (long) cookie;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == local->read_child_index) {
+                        local->read_child_returned = _gf_true;
+                }
+                __dir_entry_fop_common_cbk (frame, child_index, this,
+                                            op_ret, op_errno, NULL, NULL,
+                                            preparent, postparent, NULL, NULL);
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
+
+        return 0;
 }
 
 
 int32_t
 afr_unlink_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk,	
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->unlink,
-					   &local->loc);
-			
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->unlink,
+                                           &local->loc, local->xflag,
+                                           local->xdata_req);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int32_t
 afr_unlink_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = frame->local;
+        afr_local_t * local = frame->local;
+
+        local->transaction.unwind (frame, this);
 
-	local->transaction.unwind (frame, this);
+        AFR_STACK_DESTROY (frame);
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+        return 0;
 }
 
 
 int32_t
 afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc)
+            loc_t *loc, int xflag, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc, loc);
-
-	local->transaction.fop    = afr_unlink_wind;
-	local->transaction.done   = afr_unlink_done;
-	local->transaction.unwind = afr_unlink_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-	local->transaction.pending  = AFR_ENTRY_PENDING;
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        QUORUM_CHECK(unlink,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        loc_copy (&local->loc, loc);
+        local->xflag = xflag;
+        if (xdata)
+                local->xdata_req = dict_ref (xdata);
+
+        local->op = GF_FOP_UNLINK;
+        local->transaction.fop    = afr_unlink_wind;
+        local->transaction.done   = afr_unlink_done;
+        local->transaction.unwind = afr_unlink_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-
-	return 0;
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (unlink, frame, -1, op_errno,
+                                  NULL, NULL, NULL);
+        }
+
+        return 0;
 }
 
 /* }}} */
@@ -1462,325 +1755,208 @@ out:
 int
 afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame)
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno);
-
-	return 0;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (rmdir, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  &local->cont.dir_fop.preparent,
+                                  &local->cont.dir_fop.postparent,
+                                  NULL);
+        }
+
+        return 0;
 }
 
 
 int
-afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		    int32_t op_ret, int32_t op_errno)
+afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-	int need_unwind = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
-
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count)
-				need_unwind = 1;
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        afr_local_t *   local = NULL;
+        int call_count  = -1;
+        int child_index = (long) cookie;
+        int read_child  = 0;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+                if (afr_fop_failed (op_ret, op_errno) && (op_errno != ENOTEMPTY))
+                        afr_transaction_fop_failed (frame, this, child_index);
+                local->op_errno = op_errno;
+                local->child_errno[child_index] = op_errno;
+                if (op_ret > -1)
+                        __dir_entry_fop_common_cbk (frame, child_index, this,
+                                                   op_ret, op_errno, NULL, NULL,
+                                                   preparent, postparent, NULL,
+                                                   NULL);
+
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                afr_dir_fop_done (frame, this);
+
+        return 0;
 }
 
 
 int
 afr_rmdir_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk,	
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->rmdir,
-					   &local->loc);
-
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->rmdir,
+                                           &local->loc, local->cont.rmdir.flags,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int
 afr_rmdir_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = frame->local;
+        afr_local_t * local = frame->local;
 
-	local->transaction.unwind (frame, this);
+        local->transaction.unwind (frame, this);
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
 }
 
 
 int
 afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-	
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc, loc);
-
-	local->transaction.fop    = afr_rmdir_wind;
-	local->transaction.done   = afr_rmdir_done;
-	local->transaction.unwind = afr_rmdir_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-	local->transaction.pending  = AFR_ENTRY_PENDING;
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-
-	return 0;
-}
-
-/* }}} */
-
-/* {{{ setdents */
-
-int32_t
-afr_setdents_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		       int32_t op_ret, int32_t op_errno)
+           loc_t *loc, int flags, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-
-	local = frame->local;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
-
-		if ((op_ret != -1) && (local->success_count == 0)) {
-			local->op_ret = op_ret;
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
-}
-
-
-int32_t
-afr_setdents_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_setdents_wind_cbk,	
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->setdents,
-					   local->fd, local->cont.setdents.flags,
-					   local->cont.setdents.entries, 
-					   local->cont.setdents.count);
-			
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
-}
-
-
-int32_t
-afr_setdents_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t * local = frame->local;
-
-	AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	
-	return 0;
-}
-
-
-int32_t
-afr_setdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, int32_t flags, dir_entry_t *entries, int32_t count)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	frame->local = local;
-
-	local->fd = fd_ref (fd);
-
-	local->cont.setdents.flags   = flags;
-	local->cont.setdents.entries = entries;
-	local->cont.setdents.count   = count;
-
-	local->transaction.fop  = afr_setdents_wind;
-	local->transaction.done = afr_setdents_done;
-
-	local->transaction.basename = NULL;
-	local->transaction.pending  = AFR_ENTRY_PENDING;
-
-	afr_transaction (frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = 0;
+        int                     nlockee                 = 0;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv = this->private;
+
+        QUORUM_CHECK(rmdir,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->cont.rmdir.flags = flags;
+        loc_copy (&local->loc, loc);
+
+        local->op = GF_FOP_RMDIR;
+        local->transaction.fop    = afr_rmdir_wind;
+        local->transaction.done   = afr_rmdir_done;
+        local->transaction.unwind = afr_rmdir_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->loc,
+                                     NULL,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
+
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+        }
 
-	return 0;
+        return 0;
 }
 
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index e6e8a5e79..02f0a3682 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_WRITE_H__
@@ -22,38 +13,35 @@
 
 int32_t
 afr_create (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t flags, mode_t mode, fd_t *fd);
+            loc_t *loc, int32_t flags, mode_t mode,
+            mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev);
+	   loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode);
+	   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc);
+	    loc_t *loc, int xflag, dict_t *xdata);
 
 int32_t
 afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc);
+	   loc_t *loc, int flags, dict_t *xdata);
 
 int32_t
 afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc);
+	  loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int32_t
 afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc);
+	    loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
-int32_t
+int
 afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc);
-
-int32_t
-afr_setdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, int32_t flags, dir_entry_t *entries, int32_t count);
+	     const char *linkpath, loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index a6c99ec05..e06e3b2f2 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -44,12 +35,9 @@
 #include "compat-errno.h"
 #include "compat.h"
 
-#include "afr.h"
-
-
 /**
  * Common algorithm for inode read calls:
- * 
+ *
  * - Try the fop on the first child that is up
  * - if we have failed due to ENOTCONN:
  *     try the next child
@@ -61,93 +49,115 @@
 
 int32_t
 afr_access_cbk (call_frame_t *frame, void *cookie,
-		xlator_t *this, int32_t op_ret, int32_t op_errno)
+                xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
-
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-
-	priv     = this->private;
-	children = priv->children;
-
-	local = frame->local;
-
-	if (op_ret == -1) {
-		last_tried = local->cont.access.last_tried;
-
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try    = ++local->cont.access.last_tried;
-
-		unwind = 0;
-
-		STACK_WIND_COOKIE (frame, afr_access_cbk,
-				   (void *) (long) this_try,
-				   children[this_try], 
-				   children[this_try]->fops->access,
-				   &local->loc, local->cont.access.mask);
-	}
+        afr_private_t * priv            = NULL;
+        afr_local_t *   local           = NULL;
+        xlator_t **     children        = NULL;
+        int             unwind          = 1;
+        int32_t         *last_index     = NULL;
+        int32_t         next_call_child = -1;
+        int32_t         read_child      = -1;
+        int32_t         *fresh_children  = NULL;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+
+        read_child = (long) cookie;
+
+        if (op_ret == -1) {
+                last_index = &local->cont.access.last_index;
+                fresh_children = local->fresh_children;
+                next_call_child = afr_next_call_child (fresh_children,
+                                                       local->child_up,
+                                                       priv->child_count,
+                                                       last_index, read_child);
+                if (next_call_child < 0)
+                        goto out;
+
+                unwind = 0;
+
+                STACK_WIND_COOKIE (frame, afr_access_cbk,
+                                   (void *) (long) read_child,
+                                   children[next_call_child],
+                                   children[next_call_child]->fops->access,
+                                   &local->loc, local->cont.access.mask,
+                                   NULL);
+        }
 
 out:
-	if (unwind) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
+        if (unwind) {
+                AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int32_t
-afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask)
+afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
-
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
-
-	children = priv->children;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	call_child = afr_first_up_child (priv);
-	if (call_child == -1) {
-		op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR,
-			"no child is up :(");
-		goto out;
-	}
-
-	local->cont.access.last_tried = call_child;
-	loc_copy (&local->loc, loc);
-	local->cont.access.mask       = mask;
-
-	STACK_WIND_COOKIE (frame, afr_access_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->access,
-			   loc, mask);
-
-	op_ret = 0;
+        afr_private_t   *priv      = NULL;
+        xlator_t        **children = NULL;
+        int             call_child = 0;
+        afr_local_t     *local     = NULL;
+        int32_t         op_errno   = 0;
+        int32_t         read_child = -1;
+        int             ret        = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv     = this->private;
+        VALIDATE_OR_GOTO (priv->children, out);
+
+        children = priv->children;
+
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+
+        read_child = afr_inode_get_read_ctx (this, loc->inode,
+                                             local->fresh_children);
+        ret = afr_get_call_child (this, local->child_up, read_child,
+                                     local->fresh_children,
+                                     &call_child,
+                                     &local->cont.access.last_index);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
+
+        loc_copy (&local->loc, loc);
+        local->cont.access.mask = mask;
+
+        STACK_WIND_COOKIE (frame, afr_access_cbk,
+                           (void *) (long) call_child,
+                           children[call_child],
+                           children[call_child]->fops->access,
+                           loc, mask, xdata);
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
+        if (ret < 0)
+                AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+        return 0;
 }
 
 
@@ -157,424 +167,1629 @@ out:
 
 int32_t
 afr_stat_cbk (call_frame_t *frame, void *cookie,
-	      xlator_t *this, int32_t op_ret, int32_t op_errno,
-	      struct stat *buf)
+              xlator_t *this, int32_t op_ret, int32_t op_errno,
+              struct iatt *buf, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
-
-	int deitransform_child = -1;
+        afr_private_t * priv            = NULL;
+        afr_local_t *   local           = NULL;
+        xlator_t **     children        = NULL;
+        int             unwind          = 1;
+        int32_t         *last_index     = NULL;
+        int32_t         next_call_child = -1;
+        int32_t         read_child      = -1;
+        int32_t         *fresh_children  = NULL;
+
+        priv     = this->private;
+        children = priv->children;
+
+        read_child = (long) cookie;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                last_index = &local->cont.stat.last_index;
+                fresh_children = local->fresh_children;
+                next_call_child = afr_next_call_child (fresh_children,
+                                                       local->child_up,
+                                                       priv->child_count,
+                                                       last_index, read_child);
+                if (next_call_child < 0)
+                        goto out;
+
+                unwind = 0;
+
+                STACK_WIND_COOKIE (frame, afr_stat_cbk,
+                                   (void *) (long) read_child,
+                                   children[next_call_child],
+                                   children[next_call_child]->fops->stat,
+                                   &local->loc, NULL);
+        }
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
+out:
+        if (unwind) {
+                AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
+        }
 
-	priv     = this->private;
-	children = priv->children;
+        return 0;
+}
 
-	deitransform_child = (long) cookie;
 
-	local = frame->local;
+int32_t
+afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        afr_private_t   *priv      = NULL;
+        afr_local_t     *local     = NULL;
+        xlator_t        **children = NULL;
+        int             call_child = 0;
+        int32_t         op_errno   = 0;
+        int32_t         read_child = -1;
+        int             ret        = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv     = this->private;
+        VALIDATE_OR_GOTO (priv->children, out);
+
+        children = priv->children;
+
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        read_child = afr_inode_get_read_ctx (this, loc->inode,
+                                             local->fresh_children);
+        ret = afr_get_call_child (this, local->child_up, read_child,
+                                     local->fresh_children,
+                                     &call_child,
+                                     &local->cont.stat.last_index);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
+        loc_copy (&local->loc, loc);
+
+        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) call_child,
+                           children[call_child],
+                           children[call_child]->fops->stat,
+                           loc, xdata);
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
 
-	if (op_ret == -1) {
-	retry:
-		last_tried = local->cont.stat.last_tried;
+        return 0;
+}
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.stat.last_tried;
 
-		if (this_try == deitransform_child) {
-			goto retry;
-		}
+/* }}} */
 
-		unwind = 0;
+/* {{{ fstat */
 
-		STACK_WIND_COOKIE (frame, afr_stat_cbk,
-				   (void *) (long) deitransform_child,
-				   children[this_try], 
-				   children[this_try]->fops->stat,
-				   &local->loc);
-	}
+int32_t
+afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
+{
+        afr_private_t   *priv           = NULL;
+        afr_local_t     *local          = NULL;
+        xlator_t        **children      = NULL;
+        int             unwind          = 1;
+        int32_t         *last_index     = NULL;
+        int32_t         next_call_child = -1;
+        int32_t         read_child      = -1;
+        int32_t         *fresh_children  = NULL;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+
+        read_child = (long) cookie;
+
+        if (op_ret == -1) {
+                last_index = &local->cont.fstat.last_index;
+                fresh_children = local->fresh_children;
+                next_call_child = afr_next_call_child (fresh_children,
+                                                       local->child_up,
+                                                       priv->child_count,
+                                                       last_index, read_child);
+                if (next_call_child < 0)
+                        goto out;
+
+                unwind = 0;
+
+                STACK_WIND_COOKIE (frame, afr_fstat_cbk,
+                                   (void *) (long) read_child,
+                                   children[next_call_child],
+                                   children[next_call_child]->fops->fstat,
+                                   local->fd, NULL);
+        }
 
 out:
-	if (unwind) {
-		if (op_ret != -1)
-			buf->st_ino = local->cont.stat.ino;
+        if (unwind) {
+                AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+        }
 
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, buf);
-	}
-
-	return 0;
+        return 0;
 }
 
 
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc)
+afr_fstat (call_frame_t *frame, xlator_t *this,
+           fd_t *fd, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	afr_local_t   * local      = NULL;
-	xlator_t **     children   = NULL;
+        afr_private_t   *priv      = NULL;
+        afr_local_t     *local     = NULL;
+        xlator_t        **children = NULL;
+        int             call_child = 0;
+        int32_t         op_errno   = 0;
+        int32_t         read_child = 0;
+        int             ret        = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (fd, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv     = this->private;
+        VALIDATE_OR_GOTO (priv->children, out);
+
+        children = priv->children;
+
+        VALIDATE_OR_GOTO (fd->inode, out);
 
-	int             call_child = 0;
+        AFR_SBRAIN_CHECK_FD (fd, out);
 
-	int32_t         op_ret     = -1;
-	int32_t         op_errno   = 0;
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+        local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children) {
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-	children = priv->children;
+        read_child = afr_inode_get_read_ctx (this, fd->inode,
+                                             local->fresh_children);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
 
-	frame->local = local;
 
-	call_child = afr_deitransform (loc->inode->ino, priv->child_count);
-	loc_copy (&local->loc, loc);
+        ret = afr_get_call_child (this, local->child_up, read_child,
+                                     local->fresh_children,
+                                     &call_child,
+                                     &local->cont.fstat.last_index);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
 
-	/* 
-	   if stat fails from the deitranform'd child, we try
-	   all children starting with the first one
-	*/
-	local->cont.stat.last_tried = -1;
-	local->cont.stat.ino = loc->inode->ino;
+        local->fd = fd_ref (fd);
 
-	STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) call_child,
-			   children[call_child],
-			   children[call_child]->fops->stat,
-			   loc);
+        afr_open_fd_fix (fd, this);
 
-	op_ret = 0;
+        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) call_child,
+                           children[call_child],
+                           children[call_child]->fops->fstat,
+                           fd, xdata);
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+        if (ret < 0)
+                AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
-
 /* }}} */
 
-/* {{{ fstat */
+/* {{{ readlink */
 
 int32_t
-afr_fstat_cbk (call_frame_t *frame, void *cookie,
-	       xlator_t *this, int32_t op_ret, int32_t op_errno,
-	       struct stat *buf)
+afr_readlink_cbk (call_frame_t *frame, void *cookie,
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  const char *buf, struct iatt *sbuf, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+        afr_private_t * priv                  = NULL;
+        afr_local_t *   local                 = NULL;
+        xlator_t **     children              = NULL;
+        int             unwind                = 1;
+        int32_t         *last_index           = NULL;
+        int32_t         next_call_child       = -1;
+        int32_t         read_child            = -1;
+        int32_t         *fresh_children        = NULL;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+
+        read_child = (long) cookie;
+
+        if (op_ret == -1) {
+                last_index = &local->cont.readlink.last_index;
+                fresh_children = local->fresh_children;
+                next_call_child = afr_next_call_child (fresh_children,
+                                                       local->child_up,
+                                                       priv->child_count,
+                                                       last_index, read_child);
+                if (next_call_child < 0)
+                        goto out;
+
+                unwind = 0;
+                STACK_WIND_COOKIE (frame, afr_readlink_cbk,
+                                   (void *) (long) read_child,
+                                   children[next_call_child],
+                                   children[next_call_child]->fops->readlink,
+                                   &local->loc,
+                                   local->cont.readlink.size, NULL);
+        }
 
-	int deitransform_child = -1;
+out:
+        if (unwind) {
+                AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf,
+                                  xdata);
+        }
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
+        return 0;
+}
 
-	priv     = this->private;
-	children = priv->children;
 
-	deitransform_child = (long) cookie;
+int32_t
+afr_readlink (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, size_t size, dict_t *xdata)
+{
+        afr_private_t   *priv      = NULL;
+        xlator_t        **children = NULL;
+        int             call_child = 0;
+        afr_local_t     *local     = NULL;
+        int32_t         op_errno   = 0;
+        int32_t         read_child = -1;
+        int             ret        = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv     = this->private;
+        VALIDATE_OR_GOTO (priv->children, out);
+
+        children = priv->children;
+
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+        read_child = afr_inode_get_read_ctx (this, loc->inode,
+                                             local->fresh_children);
+        ret = afr_get_call_child (this, local->child_up, read_child,
+                                     local->fresh_children,
+                                     &call_child,
+                                     &local->cont.readlink.last_index);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
+
+        loc_copy (&local->loc, loc);
+
+        local->cont.readlink.size       = size;
+
+        STACK_WIND_COOKIE (frame, afr_readlink_cbk,
+                           (void *) (long) call_child,
+                           children[call_child],
+                           children[call_child]->fops->readlink,
+                           loc, size, xdata);
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
-	local = frame->local;
 
-	if (op_ret == -1) {
-	retry:
-		last_tried = local->cont.fstat.last_tried;
+/* }}} */
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try   = ++local->cont.fstat.last_tried;
+/* {{{ getxattr */
 
-		if (this_try == deitransform_child) {
-			/* 
-			   skip the deitransform'd child since if we are here
-			   we must have already tried that child
-			*/
-			goto retry;
-		}
-	       
+struct _xattr_key {
+        char *key;
+        struct list_head list;
+};
 
-		unwind = 0;
 
-		STACK_WIND_COOKIE (frame, afr_fstat_cbk,
-				   (void *) (long) deitransform_child,
-				   children[this_try], 
-				   children[this_try]->fops->fstat,
-				   local->fd);
-	}
+int
+__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
+                     void *data)
+{
+        struct list_head *  list  = data;
+        struct _xattr_key * xkey  = NULL;
 
-out:
-	if (unwind) {
-		if (op_ret != -1)
-			buf->st_ino = local->cont.fstat.ino;
+        if (!strncmp (key, AFR_XATTR_PREFIX,
+                      strlen (AFR_XATTR_PREFIX))) {
+
+                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
+                if (!xkey)
+                        return -1;
 
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, buf);
-	}
+                xkey->key = key;
+                INIT_LIST_HEAD (&xkey->list);
 
-	return 0;
+                list_add_tail (&xkey->list, list);
+        }
+        return 0;
 }
 
 
-int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd)
+void
+__filter_xattrs (dict_t *dict)
 {
-	afr_private_t * priv       = NULL;
-	afr_local_t   * local      = NULL;
-	xlator_t **     children   = NULL;
+        struct list_head   keys = {0,};
+        struct _xattr_key *key  = NULL;
+        struct _xattr_key *tmp  = NULL;
 
-	int             call_child = 0;
+        INIT_LIST_HEAD (&keys);
 
-	int32_t         op_ret     = -1;
-	int32_t         op_errno   = 0;
+        dict_foreach (dict, __gather_xattr_keys,
+                      (void *) &keys);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (fd, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        list_for_each_entry_safe (key, tmp, &keys, list) {
+                dict_del (dict, key->key);
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+                list_del_init (&key->list);
 
-	children = priv->children;
+                GF_FREE (key);
+        }
+}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
 
-	frame->local = local;
 
-	VALIDATE_OR_GOTO (fd->inode, out);
+int32_t
+afr_getxattr_cbk (call_frame_t *frame, void *cookie,
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  dict_t *dict, dict_t *xdata)
+{
+        afr_private_t * priv            = NULL;
+        afr_local_t *   local           = NULL;
+        xlator_t **     children        = NULL;
+        int             unwind          = 1;
+        int32_t         *last_index     = NULL;
+        int32_t         next_call_child = -1;
+        int32_t         read_child      = -1;
+        int32_t         *fresh_children  = NULL;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+
+        read_child = (long) cookie;
+
+        if (op_ret == -1) {
+                last_index = &local->cont.getxattr.last_index;
+                fresh_children = local->fresh_children;
+                next_call_child = afr_next_call_child (fresh_children,
+                                                       local->child_up,
+                                                       priv->child_count,
+                                                       last_index, read_child);
+                if (next_call_child < 0)
+                        goto out;
+
+                unwind = 0;
+                STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
+                                   (void *) (long) read_child,
+                                   children[next_call_child],
+                                   children[next_call_child]->fops->getxattr,
+                                   &local->loc,
+                                   local->cont.getxattr.name,
+                                   NULL);
+        }
 
-	call_child = afr_deitransform (fd->inode->ino, priv->child_count);
+out:
+        if (unwind) {
+                if (op_ret >= 0 && dict)
+                        __filter_xattrs (dict);
 
-	/* 
-	   if fstat fails from the deitranform'd child, we try
-	   all children starting with the first one
-	*/
-	local->cont.fstat.last_tried = -1;
-	local->cont.fstat.ino = fd->inode->ino;
-	local->fd = fd_ref (fd);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+        }
 
-	STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) call_child,
-			   children[call_child],
-			   children[call_child]->fops->fstat,
-			   fd);
+        return 0;
+}
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+int32_t
+afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
+                     dict_t *dict, dict_t *xdata)
 
-	return 0;
+{
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
-/* }}} */
-
-/* {{{ readlink */
+int32_t
+afr_fgetxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *dict, dict_t *xdata)
+{
+        afr_local_t     *local                  = NULL;
+        afr_private_t   *priv                   = NULL;
+        xlator_t        **children              = NULL;
+        dict_t          *xattr                  = NULL;
+        char            *tmp_report             = NULL;
+        char            lk_summary[1024]        = {0,};
+        int             serz_len                = 0;
+        int32_t         callcnt                 = 0;
+        long int        cky                     = 0;
+        int             ret                     = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->child_errno[cky] = op_errno;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+                if (local->dict) {
+                        ret = dict_get_str (dict, local->cont.getxattr.name,
+                                            &tmp_report);
+                        if (ret)
+                                goto unlock;
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
+                        if (ret)
+                                goto unlock;
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                xattr = dict_new ();
+                if (!xattr) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       lk_summary,
+                                                       &serz_len, '\n');
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error serializing dictionary");
+                        goto unwind;
+                }
+                if (serz_len == -1)
+                        snprintf (lk_summary, sizeof (lk_summary),
+                                  "No locks cleared.");
+                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
+                                       gf_strdup (lk_summary));
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting dictionary");
+                        goto unwind;
+                }
+
+        unwind:
+                // Updating child_errno with more recent 'events'
+                local->child_errno[cky] = op_errno;
+                op_errno = afr_resultant_errno_get (NULL, local->child_errno,
+                                                    priv->child_count);
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
+                                  xdata);
+
+                if (xattr)
+                        dict_unref (xattr);
+        }
+
+        return ret;
+}
 
 int32_t
-afr_readlink_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  const char *buf)
+afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                        xlator_t *this, int32_t op_ret, int32_t op_errno,
+                        dict_t *dict, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+        afr_local_t     *local                  = NULL;
+        afr_private_t   *priv                   = NULL;
+        xlator_t        **children              = NULL;
+        dict_t          *xattr                  = NULL;
+        char            *tmp_report             = NULL;
+        char            lk_summary[1024]        = {0,};
+        int             serz_len                = 0;
+        int32_t         callcnt                 = 0;
+        long int        cky                     = 0;
+        int             ret                     = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->child_errno[cky] = op_errno;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+                if (local->dict) {
+                        ret = dict_get_str (dict, local->cont.getxattr.name,
+                                            &tmp_report);
+                        if (ret)
+                                goto unlock;
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
+                        if (ret)
+                                goto unlock;
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                xattr = dict_new ();
+                if (!xattr) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       lk_summary,
+                                                       &serz_len, '\n');
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error serializing dictionary");
+                        goto unwind;
+                }
+                if (serz_len == -1)
+                        snprintf (lk_summary, sizeof (lk_summary),
+                                  "No locks cleared.");
+                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
+                                       gf_strdup (lk_summary));
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting dictionary");
+                        goto unwind;
+                }
+
+        unwind:
+                // Updating child_errno with more recent 'events'
+                local->child_errno[cky] = op_errno;
+                op_errno = afr_resultant_errno_get (NULL, local->child_errno,
+                                                    priv->child_count);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+
+                if (xattr)
+                        dict_unref (xattr);
+        }
+
+        return ret;
+}
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
+/**
+ * node-uuid cbk uses next child querying mechanism
+ */
+int32_t
+afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        afr_private_t  *priv            = NULL;
+        afr_local_t    *local           = NULL;
+        xlator_t      **children        = NULL;
+        int             unwind          = 1;
+        int             curr_call_child = 0;
+
+        priv = this->private;
+        children = priv->children;
+
+        local = frame->local;
+
+        if (op_ret == -1) { /** query the _next_ child */
+
+                /**
+                 * _current_ becomes _next_
+                 * If done with all childs and yet no success; give up !
+                 */
+                curr_call_child = (int) ((long)cookie);
+                if (++curr_call_child == priv->child_count)
+                        goto unwind;
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "op_ret (-1): Re-querying afr-child (%d/%d)",
+                        curr_call_child, priv->child_count);
+
+                unwind = 0;
+                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
+                                   (void *) (long) curr_call_child,
+                                   children[curr_call_child],
+                                   children[curr_call_child]->fops->getxattr,
+                                   &local->loc,
+                                   local->cont.getxattr.name,
+                                   NULL);
+        }
+
+ unwind:
+        if (unwind)
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
+                                  NULL);
+
+        return 0;
+}
 
-	priv     = this->private;
-	children = priv->children;
+int32_t
+afr_getxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                           xlator_t *this, int32_t op_ret, int32_t op_errno,
+                           dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len == 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (getxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
+
+        return 0;
+}
 
-	local = frame->local;
+int32_t
+afr_fgetxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len <= 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
+
+        return 0;
+}
 
-	if (op_ret == -1) {
-		last_tried = local->cont.readlink.last_tried;
+int32_t
+afr_fgetxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+        int          ret            = 0;
+        char        *xattr          = NULL;
+        char        *xattr_serz     = NULL;
+        char        xattr_cky[1024] = {0,};
+        dict_t      *nxattr         = NULL;
+        long         cky            = 0;
+        int32_t      padding        = 0;
+        int32_t      tlen           = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (!dict || (op_ret < 0))
+                        goto out;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+
+                if (local->dict) {
+                        ret = dict_get_str (dict,
+                                            local->cont.getxattr.name,
+                                            &xattr);
+                        if (ret)
+                                goto out;
+
+                        xattr = gf_strdup (xattr);
+
+                        (void)snprintf (xattr_cky, 1024, "%s-%ld",
+                                        local->cont.getxattr.name, cky);
+                        ret = dict_set_dynstr (local->dict,
+                                               xattr_cky, xattr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Cannot set xattr cookie key");
+                                goto out;
+                        }
+
+                        local->cont.getxattr.xattr_len
+                                += strlen (xattr) + 1;
+                }
+        }
+out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->cont.getxattr.xattr_len)
+                        goto unwind;
+
+                nxattr = dict_new ();
+                if (!nxattr)
+                        goto unwind;
+
+                /* extra bytes for decorations (brackets and <>'s) */
+                padding += strlen (this->name)
+                        + strlen (AFR_PATHINFO_HEADER) + 4;
+                local->cont.getxattr.xattr_len += (padding + 2);
+
+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
+                                        sizeof (char), gf_common_mt_char);
+
+                if (!xattr_serz)
+                        goto unwind;
+
+                /* the xlator info */
+                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
+                                this->name);
+
+                /* actual series of pathinfo */
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       xattr_serz
+                                                       + strlen (xattr_serz),
+                                                       &tlen, ' ');
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
+                                " dictionary");
+                        goto unwind;
+                }
+
+                /* closing part */
+                *(xattr_serz + padding + tlen) = ')';
+                *(xattr_serz + padding + tlen + 1) = '\0';
+
+                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
+                                       xattr_serz);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
+                                " key in dict");
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, nxattr,
+                                  xdata);
+
+                if (nxattr)
+                        dict_unref (nxattr);
+        }
+
+        return ret;
+}
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.readlink.last_tried;
+int32_t
+afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
+                           xlator_t *this, int32_t op_ret, int32_t op_errno,
+                           dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+        int          ret            = 0;
+        char        *xattr          = NULL;
+        char        *xattr_serz     = NULL;
+        char        xattr_cky[1024] = {0,};
+        dict_t      *nxattr         = NULL;
+        long         cky            = 0;
+        int32_t      padding        = 0;
+        int32_t      tlen           = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+                {
+                        callcnt = --local->call_count;
+
+                        if (!dict || (op_ret < 0))
+                                goto out;
+
+                        if (!local->dict)
+                                local->dict = dict_new ();
+
+                        if (local->dict) {
+                                ret = dict_get_str (dict,
+                                                    local->cont.getxattr.name,
+                                                    &xattr);
+                                if (ret)
+                                        goto out;
+
+                                xattr = gf_strdup (xattr);
+
+                                (void)snprintf (xattr_cky, 1024, "%s-%ld",
+                                                local->cont.getxattr.name, cky);
+                                ret = dict_set_dynstr (local->dict,
+                                                       xattr_cky, xattr);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Cannot set xattr cookie key");
+                                        goto out;
+                                }
+
+                                local->cont.getxattr.xattr_len += strlen (xattr) + 1;
+                        }
+                }
+ out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->cont.getxattr.xattr_len)
+                        goto unwind;
+
+                nxattr = dict_new ();
+                if (!nxattr)
+                        goto unwind;
+
+                /* extra bytes for decorations (brackets and <>'s) */
+                padding += strlen (this->name) + strlen (AFR_PATHINFO_HEADER) + 4;
+                local->cont.getxattr.xattr_len += (padding + 2);
+
+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
+                                        sizeof (char), gf_common_mt_char);
+
+                if (!xattr_serz)
+                        goto unwind;
+
+                /* the xlator info */
+                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
+                                this->name);
+
+                /* actual series of pathinfo */
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       xattr_serz + strlen (xattr_serz),
+                                                       &tlen, ' ');
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
+                                " dictionary");
+                        goto unwind;
+                }
+
+                /* closing part */
+                *(xattr_serz + padding + tlen) = ')';
+                *(xattr_serz + padding + tlen + 1) = '\0';
+
+                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
+                                       xattr_serz);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
+                                " key in dict");
+
+        unwind:
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, nxattr,
+                                  xdata);
+
+                if (nxattr)
+                        dict_unref (nxattr);
+        }
+
+        return ret;
+}
 
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-				   (void *) (long) this_try,
-				   children[this_try], 
-				   children[this_try]->fops->readlink,
-				   &local->loc,
-				   local->cont.readlink.size);
-	}
+static int
+afr_aggregate_stime_xattr (dict_t *this, char *key, data_t *value, void *data)
+{
+        int ret = 0;
 
-out:
-	if (unwind) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, buf);
-	}
+        if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+                ret = gf_get_min_stime (THIS, data, key, value);
 
-	return 0;
+        return ret;
 }
 
-
 int32_t
-afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size)
+afr_common_getxattr_stime_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *dict, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
-
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
 
-	children = priv->children;
+        local = frame->local;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
 
-	frame->local = local;
+                if (!dict || (op_ret < 0)) {
+                        local->op_errno = op_errno;
+                        goto cleanup;
+                }
 
-	call_child = afr_first_up_child (priv);
-	if (call_child == -1) {
-		op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR,
-			"no child is up :(");
-		goto out;
-	}
+                if (!local->dict)
+                        local->dict = dict_copy_with_ref (dict, NULL);
+                else
+                        dict_foreach (dict, afr_aggregate_stime_xattr,
+                                      local->dict);
+                local->op_ret = 0;
+        }
 
-	local->cont.readlink.last_tried = call_child;
-	loc_copy (&local->loc, loc);
-	local->cont.readlink.size       = size;
+cleanup:
+        UNLOCK (&frame->lock);
 
-	STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->readlink,
-			   loc, size);
+        if (!callcnt) {
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+                                  local->op_errno, local->dict, xdata);
+        }
 
-	op_ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
-	return 0;
+        return 0;
 }
 
 
-/* }}} */
-
-/* {{{ getxattr */
-
-int32_t
-afr_getxattr_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *dict)
+static gf_boolean_t
+afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,
+                      gf_boolean_t is_fgetxattr)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+        gf_boolean_t    is_spl = _gf_true;
+
+        GF_ASSERT (cbk);
+        if (!cbk) {
+                is_spl = _gf_false;
+                goto out;
+        }
+
+        if (!strcmp (name, GF_XATTR_PATHINFO_KEY)) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_pathinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_pathinfo_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
+                             strlen (GF_XATTR_CLRLK_CMD))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_clrlk_cbk;
+                } else {
+                        *cbk = afr_getxattr_clrlk_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                             strlen (GF_XATTR_LOCKINFO_KEY))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_lockinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_lockinfo_cbk;
+                }
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+                *cbk = afr_common_getxattr_stime_cbk;
+        } else {
+                is_spl = _gf_false;
+        }
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
+out:
+        return is_spl;
+}
 
-	priv     = this->private;
-	children = priv->children;
+static void
+afr_getxattr_frm_all_children (xlator_t *this, call_frame_t *frame,
+                               const char *name, loc_t *loc,
+                               fop_getxattr_cbk_t cbk)
+{
+        afr_private_t   *priv           = NULL;
+        afr_local_t     *local          = NULL;
+        xlator_t        **children      = NULL;
+        int             i               = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        local->call_count = priv->child_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                STACK_WIND_COOKIE (frame, cbk,
+                                   (void *) (long) i,
+                                   children[i], children[i]->fops->getxattr,
+                                   loc, name, NULL);
+        }
+        return;
+}
 
-	local = frame->local;
+int32_t
+afr_getxattr (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, const char *name, dict_t *xdata)
+{
+        afr_private_t           *priv         = NULL;
+        xlator_t                **children    = NULL;
+        int                     call_child    = 0;
+        afr_local_t             *local        = NULL;
+        xlator_list_t           *trav         = NULL;
+        xlator_t                **sub_volumes = NULL;
+        int                     i             = 0;
+        int32_t                 op_errno      = 0;
+        int32_t                 read_child    = -1;
+        int                     ret           = -1;
+        fop_getxattr_cbk_t      cbk           = NULL;
+        int                     afr_xtime_gauge[MCNT_MAX] = {0,};
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv     = this->private;
+        VALIDATE_OR_GOTO (priv->children, out);
+
+        children = priv->children;
+
+        AFR_SBRAIN_CHECK_LOC (loc, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        loc_copy (&local->loc, loc);
+        if (!name)
+                goto no_name;
+
+        local->cont.getxattr.name = gf_strdup (name);
+
+        if (!strncmp (name, AFR_XATTR_PREFIX,
+                      strlen (AFR_XATTR_PREFIX))) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: no data present for key %s",
+                        loc->path, name);
+                op_errno = ENODATA;
+                goto out;
+        }
+        if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+
+                local->marker.call_count = priv->child_count;
+
+                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
+                for (i = 0, trav = this->children; trav ;
+                     trav = trav->next, i++) {
+
+                        *(sub_volumes + i)  = trav->xlator;
+                }
+
+                if (cluster_getmarkerattr (frame, this, loc, name,
+                                           local, afr_getxattr_unwind,
+                                           sub_volumes,
+                                           priv->child_count,
+                                           MARKER_UUID_TYPE,
+                                           marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
+
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%s: failed to get marker attr (%s)",
+                                loc->path, name);
+                        op_errno = EINVAL;
+                        goto out;
+                }
+
+                return 0;
+        }
+
+        /*
+         * if we are doing getxattr with pathinfo as the key then we
+         * collect information from all childs
+         */
+        if (afr_is_special_xattr (name, &cbk, 0)) {
+                afr_getxattr_frm_all_children (this, frame, name,
+                                               loc, cbk);
+                return 0;
+        }
+
+        if (XATTR_IS_NODE_UUID (name)) {
+                i = 0;
+                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
+                                   (void *) (long) i,
+                                   children[i],
+                                   children[i]->fops->getxattr,
+                                   loc, name, xdata);
+                return 0;
+        }
+
+        if (*priv->vol_uuid) {
+                if ((match_uuid_local (name, priv->vol_uuid) == 0)
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                        local->marker.call_count = priv->child_count;
+
+                        sub_volumes = alloca ( priv->child_count
+                                               * sizeof (xlator_t *));
+                        for (i = 0, trav = this->children; trav ;
+                             trav = trav->next, i++) {
+
+                                *(sub_volumes + i)  = trav->xlator;
+
+                        }
+
+                        /* don't err out on getting ENOTCONN (brick down)
+                         * from a subset of the bricks
+                         */
+                        memcpy (afr_xtime_gauge, marker_xtime_default_gauge,
+                                sizeof (afr_xtime_gauge));
+                        afr_xtime_gauge[MCNT_NOTFOUND] = 0;
+                        afr_xtime_gauge[MCNT_ENOTCONN] = 0;
+                        if (cluster_getmarkerattr (frame, this, loc,
+                                                   name, local,
+                                                   afr_getxattr_unwind,
+                                                   sub_volumes,
+                                                   priv->child_count,
+                                                   MARKER_XTIME_TYPE,
+                                                   afr_xtime_gauge,
+                                                   priv->vol_uuid)) {
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "%s: failed to get marker attr (%s)",
+                                        loc->path, name);
+                                op_errno = EINVAL;
+                                goto out;
+                        }
+
+                        return 0;
+                }
+        }
+
+no_name:
+        local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        read_child = afr_inode_get_read_ctx (this, loc->inode,
+                                             local->fresh_children);
+        ret = afr_get_call_child (this, local->child_up, read_child,
+                                     local->fresh_children,
+                                     &call_child,
+                                     &local->cont.getxattr.last_index);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
+
+        STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
+                           (void *) (long) call_child,
+                           children[call_child],
+                           children[call_child]->fops->getxattr,
+                           loc, name, xdata);
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
 
-	if (op_ret == -1) {
-		last_tried = local->cont.getxattr.last_tried;
+/* {{{ fgetxattr */
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.getxattr.last_tried;
 
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-				   (void *) (long) this_try,
-				   children[this_try], 
-				   children[this_try]->fops->getxattr,
-				   &local->loc,
-				   local->cont.getxattr.name);
-	}
+int32_t
+afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
+                   xlator_t *this, int32_t op_ret, int32_t op_errno,
+                   dict_t *dict, dict_t *xdata)
+{
+        afr_private_t * priv            = NULL;
+        afr_local_t *   local           = NULL;
+        xlator_t **     children        = NULL;
+        int             unwind          = 1;
+        int32_t         *last_index     = NULL;
+        int32_t         next_call_child = -1;
+        int32_t         read_child      = -1;
+        int32_t         *fresh_children  = NULL;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+
+        read_child = (long) cookie;
+
+        if (op_ret == -1) {
+                last_index = &local->cont.getxattr.last_index;
+                fresh_children = local->fresh_children;
+                next_call_child = afr_next_call_child (fresh_children,
+                                                       local->child_up,
+                                                       priv->child_count,
+                                                       last_index, read_child);
+                if (next_call_child < 0)
+                        goto out;
+
+                unwind = 0;
+                STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk,
+                                   (void *) (long) read_child,
+                                   children[next_call_child],
+                                   children[next_call_child]->fops->fgetxattr,
+                                   local->fd,
+                                   local->cont.getxattr.name,
+                                   NULL);
+        }
 
 out:
-	if (unwind) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, dict);
-	}
+        if (unwind) {
+                if (op_ret >= 0 && dict)
+                        __filter_xattrs (dict);
 
-	return 0;
-}
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict,
+                                  xdata);
+        }
 
+        return 0;
+}
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name)
-{
-	afr_private_t *   priv       = NULL;
-	xlator_t **       children   = NULL;
-	int               call_child = 0;
-	afr_local_t     * local      = NULL;
+afr_fgetxattr_unwind (call_frame_t *frame,
+                      int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+{
+        AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
 
-	children = priv->children;
+static void
+afr_fgetxattr_frm_all_children (xlator_t *this, call_frame_t *frame,
+                                const char *name, fd_t *fd,
+                                fop_fgetxattr_cbk_t cbk)
+{
+        afr_private_t   *priv           = NULL;
+        afr_local_t     *local          = NULL;
+        xlator_t        **children      = NULL;
+        int             i               = 0;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	frame->local = local;
+        priv     = this->private;
+        children = priv->children;
 
-	call_child = afr_first_up_child (priv);
-	if (call_child == -1) {
-		op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR,
-			"no child is up :(");
-		goto out;
-	}
+        local = frame->local;
+        local->call_count = priv->child_count;
 
-	local->cont.getxattr.last_tried = call_child;
-	loc_copy (&local->loc, loc);
-	if (name)
-	  local->cont.getxattr.name       = strdup (name);
+        for (i = 0; i < priv->child_count; i++) {
+                STACK_WIND_COOKIE (frame, cbk,
+                                   (void *) (long) i,
+                                   children[i], children[i]->fops->fgetxattr,
+                                   fd, name, NULL);
+        }
 
-	STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->getxattr,
-			   loc, name);
+        return;
+}
 
-	op_ret = 0;
+int32_t
+afr_fgetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, const char *name, dict_t *xdata)
+{
+        afr_private_t        *priv       = NULL;
+        xlator_t            **children   = NULL;
+        int                   call_child = 0;
+        afr_local_t          *local      = NULL;
+        int32_t               op_ret     = -1;
+        int32_t               op_errno   = 0;
+        int32_t               read_child = -1;
+        fop_fgetxattr_cbk_t   cbk        = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv     = this->private;
+        VALIDATE_OR_GOTO (priv->children, out);
+
+        children = priv->children;
+
+        AFR_SBRAIN_CHECK_FD (fd, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
+        frame->local = local;
+
+        op_ret = afr_local_init (local, priv, &op_errno);
+        if (op_ret < 0) {
+                op_errno = -op_ret;
+                goto out;
+        }
+
+        local->fd = fd_ref (fd);
+        if (name)
+                local->cont.getxattr.name = gf_strdup (name);
+
+        /* pathinfo gets handled only in getxattr(), but we need to handle
+         * lockinfo.
+         * If we are doing fgetxattr with lockinfo as the key then we
+         * collect information from all children.
+         */
+        if (afr_is_special_xattr (name, &cbk, 1)) {
+                afr_fgetxattr_frm_all_children (this, frame, name,
+                                                fd, cbk);
+                return 0;
+        }
+
+
+        local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        read_child = afr_inode_get_read_ctx (this, fd->inode,
+                                             local->fresh_children);
+        op_ret = afr_get_call_child (this, local->child_up, read_child,
+                                     local->fresh_children,
+                                     &call_child,
+                                     &local->cont.getxattr.last_index);
+        if (op_ret < 0) {
+                op_errno = -op_ret;
+                op_ret = -1;
+                goto out;
+        }
+
+        STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk,
+                           (void *) (long) call_child,
+                           children[call_child],
+                           children[call_child]->fops->fgetxattr,
+                           fd, name, xdata);
+
+        op_ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
-	return 0;
+        if (op_ret == -1) {
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, NULL,
+                                  NULL);
+        }
+        return 0;
 }
 
 
@@ -584,7 +1799,7 @@ out:
 
 /**
  * read algorithm:
- * 
+ *
  * if the user has specified a read subvolume, use it
  * otherwise -
  *   use the inode number to hash it to one of the subvolumes, and
@@ -593,129 +1808,133 @@ out:
  * if any of the above read's fail, try the children in sequence
  * beginning at the beginning
  */
- 
+
 int32_t
 afr_readv_cbk (call_frame_t *frame, void *cookie,
-	       xlator_t *this, int32_t op_ret, int32_t op_errno,
-	       struct iovec *vector, int32_t count, struct stat *buf)
+               xlator_t *this, int32_t op_ret, int32_t op_errno,
+               struct iovec *vector, int32_t count, struct iatt *buf,
+               struct iobref *iobref, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
-
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
-
-	children = priv->children;
-
-	local = frame->local;
-
-	if (op_ret == -1) {
-	retry:
-		last_tried = local->cont.readv.last_tried;
-
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.readv.last_tried;
-
-		if (this_try == priv->read_child) {
-			/* 
-			   skip the read child since if we are here
-			   we must have already tried that child
-			*/
-			goto retry;
-		}
-
-		unwind = 0;
-
-		STACK_WIND_COOKIE (frame, afr_readv_cbk,
-				   (void *) (long) this_try,
-				   children[this_try], 
-				   children[this_try]->fops->readv,
-				   local->fd, local->cont.readv.size,
-				   local->cont.readv.offset);
-	}
+        afr_private_t * priv            = NULL;
+        afr_local_t *   local           = NULL;
+        xlator_t **     children        = NULL;
+        int             unwind          = 1;
+        int32_t         *last_index     = NULL;
+        int32_t         next_call_child = -1;
+        int32_t         *fresh_children  = NULL;
+        int32_t         read_child      = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        priv     = this->private;
+        VALIDATE_OR_GOTO (priv->children, out);
+
+        children = priv->children;
+
+        local = frame->local;
+
+        read_child = (long) cookie;
+
+        if (op_ret == -1) {
+                last_index = &local->cont.readv.last_index;
+                fresh_children = local->fresh_children;
+                next_call_child = afr_next_call_child (fresh_children,
+                                                       local->child_up,
+                                                       priv->child_count,
+                                                       last_index, read_child);
+                if (next_call_child < 0)
+                        goto out;
+
+                unwind = 0;
+
+                STACK_WIND_COOKIE (frame, afr_readv_cbk,
+                                   (void *) (long) read_child,
+                                   children[next_call_child],
+                                   children[next_call_child]->fops->readv,
+                                   local->fd, local->cont.readv.size,
+                                   local->cont.readv.offset,
+                                   local->cont.readv.flags,
+                                   NULL);
+        }
 
 out:
-	if (unwind) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, vector, count, buf);
-	}
+        if (unwind) {
+                AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
+                                  vector, count, buf, iobref, xdata);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int32_t
 afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset)
+           fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	afr_local_t   * local      = NULL;
-	xlator_t **     children   = NULL;
-
-	int             call_child = 0;
-
-	int32_t         op_ret     = -1;
-	int32_t         op_errno   = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-	VALIDATE_OR_GOTO (fd, out);
-
-	priv     = this->private;
-	children = priv->children;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	frame->local = local;
-
-	if (priv->read_child != -1) {
-		call_child = priv->read_child;
-
-		/* 
-		   if read fails from the read child, we try
-		   all children starting with the first one
-		*/
-		local->cont.readv.last_tried = -1;
-	} else {
-		call_child = afr_first_up_child (priv);
-		if (call_child == -1) {
-			op_errno = ENOTCONN;
-			gf_log (this->name, GF_LOG_ERROR,
-				"no child is up :(");
-			goto out;
-		}
-
-		local->cont.readv.last_tried = call_child;
-	}
-
-	local->fd                    = fd_ref (fd);
-
-	local->cont.readv.size       = size;
-	local->cont.readv.offset     = offset;
-
-	STACK_WIND_COOKIE (frame, afr_readv_cbk,
-			   (void *) (long) call_child,
-			   children[call_child],
-			   children[call_child]->fops->readv,
-			   fd, size, offset);
-
-	op_ret = 0;
+        afr_private_t * priv       = NULL;
+        afr_local_t   * local      = NULL;
+        xlator_t **     children   = NULL;
+        int             call_child = 0;
+        int32_t         op_errno   = 0;
+        int32_t         read_child = -1;
+        int             ret        = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+        VALIDATE_OR_GOTO (fd, out);
+
+        priv     = this->private;
+        children = priv->children;
+
+        AFR_SBRAIN_CHECK_FD (fd, out);
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children);
+        ret = afr_get_call_child (this, local->child_up, read_child,
+                                     local->fresh_children,
+                                     &call_child,
+                                     &local->cont.readv.last_index);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
+
+        local->fd                    = fd_ref (fd);
+
+        local->cont.readv.size       = size;
+        local->cont.readv.offset     = offset;
+        local->cont.readv.flags      = flags;
+
+        afr_open_fd_fix (fd, this);
+
+        STACK_WIND_COOKIE (frame, afr_readv_cbk,
+                           (void *) (long) call_child,
+                           children[call_child],
+                           children[call_child]->fops->readv,
+                           fd, size, offset, flags, xdata);
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL, 0, NULL);
-	}
-	return 0;
+        if (ret < 0) {
+                AFR_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL,
+                                  NULL, NULL);
+        }
+        return 0;
 }
 
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index 6b3bd2da8..e4091a793 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_READ_H__
@@ -22,26 +13,30 @@
 
 int32_t
 afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask);
+	    loc_t *loc, int32_t mask, dict_t *xdata);
 
 int32_t
 afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc);
+	  loc_t *loc, dict_t *xdata);
 
 int32_t
 afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd);
+	   fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size);
+	      loc_t *loc, size_t size, dict_t *xdata);
 
 int32_t
 afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset);
+	   fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
 afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name);
+	      loc_t *loc, const char *name, dict_t *xdata);
+
+int32_t
+afr_fgetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, const char *name, dict_t *xdata);
 
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 267350b2c..c1ec69a55 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -46,1979 +37,2825 @@
 
 #include "afr.h"
 #include "afr-transaction.h"
+#include "afr-self-heal-common.h"
 
+void
+__inode_write_fop_cbk (call_frame_t *frame, int child_index, int read_child,
+                       xlator_t *this, int32_t *op_ret, int32_t *op_errno,
+                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t     *local = NULL;
+
+        local = frame->local;
+
+        if (afr_fop_failed (*op_ret, *op_errno)) {
+                local->child_errno[child_index] = *op_errno;
+
+                switch (local->op) {
+                case GF_FOP_TRUNCATE:
+                case GF_FOP_FTRUNCATE:
+                        if (*op_errno != EFBIG)
+                                afr_transaction_fop_failed (frame, this,
+                                                            child_index);
+                break;
+                default:
+                        afr_transaction_fop_failed (frame, this, child_index);
+                break;
+                }
+                local->op_errno = *op_errno;
+                goto out;
+        }
+
+        if ((local->success_count == 0) || (read_child == child_index)) {
+                local->op_ret              = *op_ret;
+                if (prebuf)
+                        local->cont.inode_wfop.prebuf  = *prebuf;
+                if (postbuf)
+                        local->cont.inode_wfop.postbuf = *postbuf;
+        }
+
+        local->success_count++;
+out:
+        return;
+}
 
-/* {{{ chmod */
-
+/* {{{ writev */
 
-int
-afr_chmod_unwind (call_frame_t *frame, xlator_t *this)
+void
+afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
+        afr_local_t *src_local = NULL;
+        afr_local_t *dst_local = NULL;
 
-	local = frame->local;
-	priv  = this->private;
+        src_local = src_frame->local;
+        dst_local = dst_frame->local;
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
+        dst_local->op_ret = src_local->op_ret;
+        dst_local->op_errno = src_local->op_errno;
+        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+}
 
-	if (main_frame) {
-		local->cont.chmod.buf.st_ino = local->cont.chmod.ino;
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  &local->cont.chmod.buf);
-	}
-	return 0;
+void
+afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        local = frame->local;
+
+        AFR_STACK_UNWIND (writev, frame,
+                          local->op_ret, local->op_errno,
+                          &local->cont.inode_wfop.prebuf,
+                          &local->cont.inode_wfop.postbuf,
+                          NULL);
 }
 
+call_frame_t*
+afr_transaction_detach_fop_frame (call_frame_t *frame)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *fop_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                fop_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        return fop_frame;
+}
 
 int
-afr_chmod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		    int32_t op_ret, int32_t op_errno, struct stat *buf)
+afr_transaction_writev_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        call_frame_t *fop_frame = NULL;
 
-	int call_count  = -1;
-	int child_index = (long) cookie;
-	int need_unwind = 0;
+        fop_frame = afr_transaction_detach_fop_frame (frame);
 
-	local = frame->local;
-	priv  = this->private;
+        if (fop_frame) {
+                afr_writev_copy_outvars (frame, fop_frame);
+                afr_writev_unwind (fop_frame, this);
+        }
+        return 0;
+}
 
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
+static void
+afr_writev_handle_short_writes (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int           i      = 0;
+
+        local = frame->local;
+        priv = this->private;
+        /*
+         * We already have the best case result of the writev calls staged
+         * as the return value. Any writev that returns some value less
+         * than the best case is now out of sync, so mark the fop as
+         * failed. Note that fops that have returned with errors have
+         * already been marked as failed.
+         */
+        for (i = 0; i < priv->child_count; i++) {
+                if ((!local->replies[i].valid) ||
+                    (local->replies[i].op_ret == -1))
+                        continue;
+
+                if (local->replies[i].op_ret < local->op_ret)
+                        afr_transaction_fop_failed(frame, this, i);
+        }
+}
 
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-				local->cont.chmod.buf = *buf;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
+int
+afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        afr_private_t  *priv  = NULL;
+        call_frame_t    *fop_frame = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int read_child  = 0;
+        int      ret = 0;
+        uint32_t open_fd_count = 0;
+        uint32_t write_is_append = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+
+		local->replies[child_index].valid = 1;
+		local->replies[child_index].op_ret = op_ret;
+		local->replies[child_index].op_errno = op_errno;
+
+
+		/* stage the best case return value for unwind */
+                if ((local->success_count == 0) || (op_ret > local->op_ret)) {
+                        local->op_ret              = op_ret;
+			local->op_errno		   = op_errno;
 		}
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (need_unwind)
-		afr_chmod_unwind (frame, this);
+		if (op_ret != -1) {
+                        if (xdata) {
+                                ret = dict_get_uint32 (xdata,
+                                                       GLUSTERFS_OPEN_FD_COUNT,
+                                                       &open_fd_count);
+                                if ((ret == 0) &&
+                                    (open_fd_count > local->open_fd_count)) {
+                                        local->open_fd_count = open_fd_count;
+                                        local->update_open_fd_count = _gf_true;
+                                }
+
+				write_is_append = 0;
+                                ret = dict_get_uint32 (xdata,
+                                                       GLUSTERFS_WRITE_IS_APPEND,
+                                                       &write_is_append);
+                                if (ret || !write_is_append)
+					local->append_write = _gf_false;
+                        }
 
-	call_count = afr_frame_return (frame);
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+
+                if (local->update_open_fd_count)
+                        afr_handle_open_fd_count (frame, this);
+
+                if (!local->stable_write && !local->append_write)
+			/* An appended write removes the necessity to
+			   fsync() the file. This is because self-heal
+			   has the logic to check for larger file when
+			   the xattrs are not reliably pointing at
+			   a stale file.
+			*/
+                        afr_fd_report_unstable_write (this, local->fd);
+
+                afr_writev_handle_short_writes (frame, this);
+                if (afr_any_fops_failed (local, priv)) {
+                        //Don't unwind until post-op is complete
+                        local->transaction.resume (frame, this);
+                } else {
+                /*
+                 * Generally inode-write fops do transaction.unwind then
+                 * transaction.resume, but writev needs to make sure that
+                 * delayed post-op frame is placed in fdctx before unwind
+                 * happens. This prevents the race of flush doing the
+                 * changelog wakeup first in fuse thread and then this
+                 * writev placing its delayed post-op frame in fdctx.
+                 * This helps flush make sure all the delayed post-ops are
+                 * completed.
+                 */
+
+                        fop_frame = afr_transaction_detach_fop_frame (frame);
+                        afr_writev_copy_outvars (frame, fop_frame);
+                        local->transaction.resume (frame, this);
+                        afr_writev_unwind (fop_frame, this);
+                }
+        }
+        return 0;
+}
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
+int
+afr_writev_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int i = 0;
+        int call_count = -1;
+        dict_t *xdata = NULL;
+        GF_UNUSED int     ret = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+	local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+				   gf_afr_mt_reply_t);
+	if (!local->replies) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		local->transaction.unwind(frame, this);
+		local->transaction.resume(frame, this);
+		return 0;
 	}
-	
-	return 0;
+
+        xdata = dict_new ();
+        if (xdata) {
+                ret = dict_set_uint32 (xdata, GLUSTERFS_OPEN_FD_COUNT,
+                                       sizeof (uint32_t));
+		ret = dict_set_uint32 (xdata, GLUSTERFS_WRITE_IS_APPEND,
+				       0);
+		/* Set append_write to be true speculatively. If on any
+		   server it turns not be true, we unset it in the
+		   callback.
+		*/
+		local->append_write = _gf_true;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_writev_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->writev,
+                                           local->fd,
+                                           local->cont.writev.vector,
+                                           local->cont.writev.count,
+                                           local->cont.writev.offset,
+                                           local->cont.writev.flags,
+                                           local->cont.writev.iobref,
+                                           xdata);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
 }
 
 
 int
-afr_chmod_wind (call_frame_t *frame, xlator_t *this)
+afr_writev_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	
-	int i = 0;
-	int call_count = -1;
+        afr_local_t *local = NULL;
 
-	local = frame->local;
-	priv  = this->private;
+        local = frame->local;
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
+        iobref_unref (local->cont.writev.iobref);
+        local->cont.writev.iobref = NULL;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
+        local->transaction.unwind (frame, this);
 
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_chmod_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->chmod,
-					   &local->loc, 
-					   local->cont.chmod.mode); 
-		
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        AFR_STACK_DESTROY (frame);
+
+        return 0;
 }
 
 
 int
-afr_chmod_done (call_frame_t *frame, xlator_t *this)
+afr_do_writev (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = NULL;
-
-	local = frame->local;
+        call_frame_t    *transaction_frame = NULL;
+        afr_local_t     *local             = NULL;
+        int             op_ret   = -1;
+        int             op_errno = 0;
+
+        local = frame->local;
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        transaction_frame->local = local;
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+
+        local->op = GF_FOP_WRITE;
+
+        local->success_count      = 0;
+
+        local->transaction.fop    = afr_writev_wind;
+        local->transaction.done   = afr_writev_done;
+        local->transaction.unwind = afr_transaction_writev_unwind;
+
+        local->transaction.main_frame = frame;
+        if (local->fd->flags & O_APPEND) {
+               /*
+                * Backend vfs ignores the 'offset' for append mode fd so
+                * locking just the region provided for the writev does not
+                * give consistency gurantee. The actual write may happen at a
+                * completely different range than the one provided by the
+                * offset, len in the fop. So lock the entire file.
+                */
+                local->transaction.start   = 0;
+                local->transaction.len     = 0;
+        } else {
+                local->transaction.start   = local->cont.writev.offset;
+                local->transaction.len     = iov_length (local->cont.writev.vector,
+                                                         local->cont.writev.count);
+        }
+
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL, NULL);
+        }
 
-	local->transaction.unwind (frame, this);
+        return 0;
+}
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+static void
+afr_trigger_open_fd_self_heal (fd_t *fd, xlator_t *this)
+{
+        call_frame_t    *frame   = NULL;
+        afr_local_t     *local   = NULL;
+        afr_self_heal_t *sh      = NULL;
+        char            *reason  = NULL;
+        int32_t         op_errno = 0;
+        int             ret      = 0;
+
+        if (!fd || !fd->inode || uuid_is_null (fd->inode->gfid)) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid args: "
+                                  "fd: %p, inode: %p", fd,
+                                  fd ? fd->inode : NULL);
+                goto out;
+        }
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+        ret = afr_local_init (local, this->private, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->loc.inode = inode_ref (fd->inode);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
+
+        sh    = &local->self_heal;
+        sh->do_metadata_self_heal = _gf_true;
+        if (fd->inode->ia_type == IA_IFREG)
+                sh->do_data_self_heal = _gf_true;
+        else if (fd->inode->ia_type == IA_IFDIR)
+                sh->do_entry_self_heal = _gf_true;
+
+        reason = "subvolume came online";
+        afr_launch_self_heal (frame, this, fd->inode, _gf_true,
+                              fd->inode->ia_type, reason, NULL, NULL);
+        return;
+out:
+        AFR_STACK_DESTROY (frame);
 }
 
+void
+afr_open_fd_fix (fd_t *fd, xlator_t *this)
+{
+        int           ret             = 0;
+        int           i               = 0;
+        afr_fd_ctx_t  *fd_ctx         = NULL;
+        gf_boolean_t  need_self_heal  = _gf_false;
+        int           *need_open      = NULL;
+        size_t        need_open_count = 0;
+        afr_private_t *priv           = NULL;
+
+        priv  = this->private;
+
+        if (!afr_is_fd_fixable (fd))
+                goto out;
+
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx)
+                goto out;
+
+        LOCK (&fd->lock);
+        {
+                if (fd_ctx->up_count < priv->up_count) {
+                        need_self_heal = _gf_true;
+                        fd_ctx->up_count   = priv->up_count;
+                        fd_ctx->down_count = priv->down_count;
+                }
+
+                need_open = alloca (priv->child_count * sizeof (*need_open));
+                for (i = 0; i < priv->child_count; i++) {
+                        need_open[i] = 0;
+                        if (fd_ctx->opened_on[i] != AFR_FD_NOT_OPENED)
+                                continue;
+
+                        if (!priv->child_up[i])
+                                continue;
+
+                        fd_ctx->opened_on[i] = AFR_FD_OPENING;
+
+                        need_open[i] = 1;
+                        need_open_count++;
+                }
+        }
+        UNLOCK (&fd->lock);
+        if (ret)
+                goto out;
+
+        if (need_self_heal)
+                afr_trigger_open_fd_self_heal (fd, this);
+
+        if (!need_open_count)
+                goto out;
+
+        afr_fix_open (this, fd, need_open_count, need_open);
+out:
+        return;
+}
 
-int32_t
-afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode)
+int
+afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+            struct iovec *vector, int32_t count, off_t offset,
+            uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        int ret = -1;
+        int op_errno = 0;
 
-	int ret = -1;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        priv = this->private;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
 
-	priv = this->private;
+        QUORUM_CHECK(writev,out);
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-	transaction_frame->local = local;
+        local->cont.writev.vector     = iov_dup (vector, count);
+        local->cont.writev.count      = count;
+        local->cont.writev.offset     = offset;
+        local->cont.writev.flags      = flags;
+        local->cont.writev.iobref     = iobref_ref (iobref);
 
-	local->cont.chmod.mode = mode;
-	local->cont.chmod.ino  = loc->inode->ino;
+        local->fd                = fd_ref (fd);
 
-	local->transaction.fop    = afr_chmod_wind;
-	local->transaction.done   = afr_chmod_done;
-	local->transaction.unwind = afr_chmod_unwind;
+	/* detect here, but set it in writev_wind_cbk *after* the unstable
+	   write is performed
+	*/
+	local->stable_write = !!((fd->flags|flags)&(O_SYNC|O_DSYNC));
 
-	loc_copy (&local->loc, loc);
-	
-	local->transaction.main_frame = frame;
-	local->transaction.start   = 0;
-	local->transaction.len     = 0;
-	local->transaction.pending = AFR_METADATA_PENDING;
+        afr_open_fd_fix (fd, this);
 
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        afr_do_writev (frame, this);
 
-	op_ret = 0;
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+        if (ret < 0)
+                AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
-/* }}} */
 
+/* }}} */
 
-/* {{{ fchmod */
+/* {{{ truncate */
 
 int
-afr_fchmod_unwind (call_frame_t *frame, xlator_t *this)
+afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-	priv  = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		local->cont.fchmod.buf.st_ino = local->cont.fchmod.ino;
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  &local->cont.fchmod.buf);
-	}
-	return 0;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (truncate, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+
+        return 0;
 }
 
 
 int
-afr_fchmod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		    int32_t op_ret, int32_t op_errno, struct stat *buf)
+afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_local_t *   local = NULL;
+        int child_index = (long) cookie;
+        int read_child  = 0;
+        int call_count  = -1;
 
-	int call_count  = -1;
-	int child_index = (long) cookie;
-	int need_unwind = 0;
+        local = frame->local;
 
-	local = frame->local;
-	priv  = this->private;
+        read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
 
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
 
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-				local->cont.fchmod.buf = *buf;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
+                if (op_ret != -1) {
+			if (prebuf->ia_size != postbuf->ia_size)
+				local->stable_write = _gf_false;
+                }
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+        }
+        UNLOCK (&frame->lock);
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        call_count = afr_frame_return (frame);
 
-	if (need_unwind)
-		afr_fchmod_unwind (frame, this);
+        if (call_count == 0) {
+		if (local->stable_write && afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
 
-	call_count = afr_frame_return (frame);
+                local->transaction.resume (frame, this);
+        }
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return 0;
 }
 
 
-int
-afr_fchmod_wind (call_frame_t *frame, xlator_t *this)
+int32_t
+afr_truncate_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	
-	int i = 0;
-	int call_count = -1;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_fchmod_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->fchmod,
-					   local->fd, 
-					   local->cont.fchmod.mode); 
-		
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+	local->stable_write = _gf_true;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->truncate,
+                                           &local->loc,
+                                           local->cont.truncate.offset,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int
-afr_fchmod_done (call_frame_t *frame, xlator_t *this)
+afr_truncate_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = NULL;
+        afr_local_t *local = NULL;
+
+        local = frame->local;
 
-	local = frame->local;
+        local->transaction.unwind (frame, this);
 
-	local->transaction.unwind (frame, this);
+        AFR_STACK_DESTROY (frame);
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+        return 0;
 }
 
 
-int32_t
-afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode)
+int
+afr_truncate (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, off_t offset, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
 
-	int ret = -1;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        priv = this->private;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        QUORUM_CHECK(truncate,out);
 
-	priv = this->private;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-	transaction_frame->local = local;
+        local->cont.truncate.offset  = offset;
 
-	local->cont.fchmod.mode = mode;
-	local->cont.fchmod.ino  = fd->inode->ino;
+        local->transaction.fop    = afr_truncate_wind;
+        local->transaction.done   = afr_truncate_done;
+        local->transaction.unwind = afr_truncate_unwind;
 
-	local->transaction.fop    = afr_fchmod_wind;
-	local->transaction.done   = afr_fchmod_done;
-	local->transaction.unwind = afr_fchmod_unwind;
+        loc_copy (&local->loc, loc);
 
-	local->fd = fd_ref (fd);
-	
-	local->transaction.main_frame = frame;
-	local->transaction.start   = 0;
-	local->transaction.len     = 0;
-	local->transaction.pending = AFR_METADATA_PENDING;
+        local->transaction.main_frame = frame;
+        local->transaction.start   = offset;
+        local->transaction.len     = 0;
 
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
-	op_ret = 0;
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+        }
 
-	return 0;
+        return 0;
 }
 
+
 /* }}} */
 
-/* {{{ chown */
+/* {{{ ftruncate */
+
 
 int
-afr_chown_unwind (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-	priv  = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		local->cont.chown.buf.st_ino = local->cont.chown.ino;
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  &local->cont.chown.buf);
-	}
-	return 0;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+        return 0;
 }
 
 
 int
-afr_chown_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		    int32_t op_ret, int32_t op_errno, struct stat *buf)
+afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
- 
-	int call_count  = -1;
-	int child_index = (long) cookie;
-	int need_unwind = 0;
+        afr_local_t *   local = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int read_child  = 0;
 
-	local = frame->local;
-	priv = this->private;
+        local = frame->local;
 
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
 
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-				local->cont.chown.buf = *buf;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+                if (op_ret != -1) {
+			if (prebuf->ia_size != postbuf->ia_size)
+				local->stable_write = _gf_false;
+                }
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
+        }
+        UNLOCK (&frame->lock);
 
-	if (need_unwind) {
-		local->transaction.unwind (frame, this);
-	}
+        call_count = afr_frame_return (frame);
 
-	call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+		if (local->stable_write && afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
 }
 
 
 int
-afr_chown_wind (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int i = 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+	local->stable_write = _gf_true;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->ftruncate,
+                                           local->fd,
+                                           local->cont.ftruncate.offset,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
 
-	local = frame->local;
-	priv  = this->private;
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
+int
+afr_ftruncate_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
+        local = frame->local;
 
-	local->call_count = call_count;
+        local->transaction.unwind (frame, this);
 
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_chown_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->chown,
-					   &local->loc, local->cont.chown.uid,
-					   local->cont.chown.gid); 
+        AFR_STACK_DESTROY (frame);
 
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        return 0;
 }
 
 
 int
-afr_chown_done (call_frame_t *frame, xlator_t *this)
+afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
+        call_frame_t * transaction_frame = NULL;
+        afr_local_t *  local             = NULL;
+        int op_ret   = -1;
+        int op_errno = 0;
 
-	local = frame->local;
+        local = frame->local;
 
-	local->transaction.unwind (frame, this);
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
 
-	AFR_STACK_DESTROY (frame);
+        transaction_frame->local = local;
+        frame->local = NULL;
 
-	return 0;
-}
+        local->op = GF_FOP_FTRUNCATE;
 
+        local->transaction.fop    = afr_ftruncate_wind;
+        local->transaction.done   = afr_ftruncate_done;
+        local->transaction.unwind = afr_ftruncate_unwind;
 
-int
-afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.ftruncate.offset;
+        local->transaction.len     = 0;
 
-	int ret = -1;
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
 
-	priv = this->private;
+int
+afr_ftruncate (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, off_t offset, dict_t *xdata)
+{
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        priv = this->private;
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(ftruncate,out);
 
-	transaction_frame->local = local;
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
 
-	local->cont.chown.uid  = uid;
-	local->cont.chown.gid  = gid;
-	local->cont.chown.ino  = loc->inode->ino;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-	local->transaction.fop    = afr_chown_wind;
-	local->transaction.done   = afr_chown_done;
-	local->transaction.unwind = afr_chown_unwind;
+        local->cont.ftruncate.offset  = offset;
 
-	loc_copy (&local->loc, loc);
+        local->fd = fd_ref (fd);
 
-	local->transaction.main_frame = frame;
-	local->transaction.start   = 0;
-	local->transaction.len     = 0;
-	local->transaction.pending = AFR_METADATA_PENDING;
+        afr_open_fd_fix (fd, this);
 
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        afr_do_ftruncate (frame, this);
 
-	op_ret = 0;
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+        }
 
-	return 0;
+        return 0;
 }
 
-
 /* }}} */
 
-/* {{{ chown */
+/* {{{ setattr */
 
 int
-afr_fchown_unwind (call_frame_t *frame, xlator_t *this)
+afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-	priv  = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		local->cont.fchown.buf.st_ino = local->cont.fchown.ino;
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  &local->cont.fchown.buf);
-	}
-	return 0;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (setattr, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+
+        return 0;
 }
 
 
 int
-afr_fchown_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		    int32_t op_ret, int32_t op_errno, struct stat *buf)
+afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
- 
-	int call_count  = -1;
-	int child_index = (long) cookie;
-	int need_unwind = 0;
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int read_child  = 0;
+        int call_count  = -1;
+        int need_unwind = 0;
 
-	local = frame->local;
-	priv = this->private;
+        local = frame->local;
+        priv  = this->private;
 
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
+        read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
 
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-				local->cont.fchown.buf = *buf;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, preop, postop,
+                                       xdata);
 
-	if (need_unwind) {
-		local->transaction.unwind (frame, this);
-	}
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
 
-	call_count = afr_frame_return (frame);
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
-}
+        call_count = afr_frame_return (frame);
 
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
 
-int
-afr_fchown_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
+        return 0;
+}
 
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_fchown_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->fchown,
-					   local->fd, local->cont.fchown.uid,
-					   local->cont.fchown.gid); 
 
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+int32_t
+afr_setattr_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->setattr,
+                                           &local->loc,
+                                           &local->cont.setattr.in_buf,
+                                           local->cont.setattr.valid,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int
-afr_fchown_done (call_frame_t *frame, xlator_t *this)
+afr_setattr_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
+        afr_local_t *local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	local->transaction.unwind (frame, this);
+        local->transaction.unwind (frame, this);
 
-	AFR_STACK_DESTROY (frame);
+        AFR_STACK_DESTROY (frame);
 
-	return 0;
+        return 0;
 }
 
 
 int
-afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid)
+afr_setattr (call_frame_t *frame, xlator_t *this,
+             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
 
-	int ret = -1;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        priv = this->private;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        QUORUM_CHECK(setattr,out);
 
-	priv = this->private;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
 
-	transaction_frame->local = local;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-	local->cont.fchown.uid  = uid;
-	local->cont.fchown.gid  = gid;
-	local->cont.fchown.ino  = fd->inode->ino;
+        local->cont.setattr.in_buf = *buf;
+        local->cont.setattr.valid  = valid;
 
-	local->transaction.fop    = afr_fchown_wind;
-	local->transaction.done   = afr_fchown_done;
-	local->transaction.unwind = afr_fchown_unwind;
+        local->transaction.fop    = afr_setattr_wind;
+        local->transaction.done   = afr_setattr_done;
+        local->transaction.unwind = afr_setattr_unwind;
 
-	local->fd = fd_ref (fd);
+        loc_copy (&local->loc, loc);
 
-	local->transaction.main_frame = frame;
-	local->transaction.start   = 0;
-	local->transaction.len     = 0;
-	local->transaction.pending = AFR_METADATA_PENDING;
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
 
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
-	op_ret = 0;
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+        }
 
-	return 0;
+        return 0;
 }
 
-/* }}} */
-
-/* {{{ writev */
+/* {{{ fsetattr */
 
 int
-afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-	priv  = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		local->cont.writev.buf.st_ino = local->cont.writev.ino;
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  &local->cont.writev.buf);
-	}
-	return 0;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+
+        return 0;
 }
 
 
 int
-afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno, struct stat *buf)
+afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int read_child  = 0;
+        int call_count  = -1;
+        int need_unwind = 0;
 
-	int child_index = (long) cookie;
-	int call_count  = -1;
-	int need_unwind = 0;
+        local = frame->local;
+        priv  = this->private;
 
-	local = frame->local;
-	priv = this->private;
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
 
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
 
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret   = op_ret;
-				local->cont.writev.buf = *buf;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, preop, postop,
+                                       xdata);
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
 
-	call_count = afr_frame_return (frame);
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
+        call_count = afr_frame_return (frame);
 
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
 }
 
 
-int
-afr_writev_wind (call_frame_t *frame, xlator_t *this)
+int32_t
+afr_fsetattr_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	
-	int i = 0;
-	int call_count = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_writev_wind_cbk, 
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->writev,
-					   local->fd, 
-					   local->cont.writev.vector,
-					   local->cont.writev.count, 
-					   local->cont.writev.offset); 
-		
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fsetattr,
+                                           local->fd,
+                                           &local->cont.fsetattr.in_buf,
+                                           local->cont.fsetattr.valid,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int
-afr_writev_done (call_frame_t *frame, xlator_t *this)
+afr_fsetattr_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
+        afr_local_t *local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	if (local->cont.writev.refs)
-		dict_unref (local->cont.writev.refs);
-	local->cont.writev.refs = NULL;
+        local->transaction.unwind (frame, this);
 
-	local->transaction.unwind (frame, this);
+        AFR_STACK_DESTROY (frame);
 
-	AFR_STACK_DESTROY (frame);
-
-	return 0;
+        return 0;
 }
 
-
 int
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, 
-	    struct iovec *vector, int32_t count, off_t offset)
+afr_fsetattr (call_frame_t *frame, xlator_t *this,
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
 
-	int ret = -1;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        priv = this->private;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
 
-	priv = this->private;
+        QUORUM_CHECK(fsetattr,out);
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	local->op = GF_FOP_WRITE;
-	local->cont.writev.vector  = iov_dup (vector, count);
-	local->cont.writev.count   = count;
-	local->cont.writev.offset  = offset;
-	local->cont.writev.ino     = fd->inode->ino;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-	if (frame->root->req_refs)
-		local->cont.writev.refs = dict_ref (frame->root->req_refs);
+        local->cont.fsetattr.in_buf = *buf;
+        local->cont.fsetattr.valid  = valid;
 
-	local->transaction.fop    = afr_writev_wind;
-	local->transaction.done   = afr_writev_done;
-	local->transaction.unwind = afr_writev_unwind;
+        local->transaction.fop    = afr_fsetattr_wind;
+        local->transaction.done   = afr_fsetattr_done;
+        local->transaction.unwind = afr_fsetattr_unwind;
 
-	local->fd                = fd_ref (fd);
+        local->fd                 = fd_ref (fd);
 
-	local->transaction.main_frame = frame;
-	if (fd->flags & O_APPEND) {
-		local->transaction.start   = 0;
-		local->transaction.len     = 0;
-	} else {
-		local->transaction.start   = offset;
-		local->transaction.len     = iov_length (vector, count);
-	}
+        afr_open_fd_fix (fd, this);
 
-	local->transaction.pending = AFR_DATA_PENDING;
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
 
-	afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
-	op_ret = 0;
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+        }
 
-	return 0;
+        return 0;
 }
 
 
-/* }}} */
+/* {{{ setxattr */
 
-/* {{{ truncate */
 
 int
-afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-	priv  = this->private;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (setxattr, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  NULL);
+        }
+        return 0;
+}
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
 
-	if (main_frame) {
-		local->cont.truncate.buf.st_ino = local->cont.truncate.ino;
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  &local->cont.truncate.buf);
-	}
-	return 0;
+int
+afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
+
+        local = frame->local;
+        priv = this->private;
+
+        LOCK (&frame->lock);
+        {
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->child_count) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
 }
 
 
 int
-afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		       int32_t op_ret, int32_t op_errno, struct stat *buf)
+afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_local_t      *local         = NULL;
+        afr_private_t    *priv          = NULL;
+        int               call_count    = -1;
+        int               i             = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->setxattr,
+                                           &local->loc,
+                                           local->cont.setxattr.dict,
+                                           local->cont.setxattr.flags,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
 
-	int child_index = (long) cookie;
-	int call_count  = -1;
-	int need_unwind = 0;
 
-	local = frame->local;
-	priv  = this->private;
+int
+afr_setxattr_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local    = frame->local;
 
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
+        local->transaction.unwind (frame, this);
 
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-				local->cont.truncate.buf = *buf;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        AFR_STACK_DESTROY (frame);
 
-	if (need_unwind)
-		local->transaction.unwind (frame, this);
+        return 0;
+}
 
-	call_count = afr_frame_return (frame);
+int
+afr_setxattr (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        afr_private_t  *priv              = NULL;
+        afr_local_t    *local             = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int             ret               = -1;
+        int             op_errno          = EINVAL;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
-}
+        VALIDATE_OR_GOTO (this, out);
 
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
+                                   op_errno, out);
 
-int32_t
-afr_truncate_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	
-	int call_count = -1;
-	int i = 0;
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
+                                   op_errno, out);
 
-	local = frame->local;
-	priv = this->private;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
+        priv = this->private;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
+        QUORUM_CHECK(setxattr,out);
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-	local->call_count = call_count;
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
 
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->truncate,
-					   &local->loc, 
-					   local->cont.truncate.offset);
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
-}
+        local->cont.setxattr.dict  = dict_ref (dict);
+        local->cont.setxattr.flags = flags;
 
+        local->transaction.fop    = afr_setxattr_wind;
+        local->transaction.done   = afr_setxattr_done;
+        local->transaction.unwind = afr_setxattr_unwind;
 
-int
-afr_truncate_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
+        loc_copy (&local->loc, loc);
 
-	local = frame->local;
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
 
-	local->transaction.unwind (frame, this);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
-	AFR_STACK_DESTROY (frame);
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+        }
 
-	return 0;
+        return 0;
 }
 
+/* {{{ fsetxattr */
+
 
 int
-afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset)
+afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
+        afr_local_t    *local         = NULL;
+        call_frame_t   *main_frame    = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (fsetxattr, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  NULL);
+        }
+        return 0;
+}
 
-	int ret = -1;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+int
+afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        local = frame->local;
+        priv = this->private;
 
-	priv = this->private;
+        LOCK (&frame->lock);
+        {
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->child_count) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        call_count = afr_frame_return (frame);
 
-	transaction_frame->local = local;
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
 
-	local->op_ret = -1;
+        return 0;
+}
 
-	local->cont.truncate.offset  = offset;
-	local->cont.truncate.ino     = loc->inode->ino;
 
-	local->transaction.fop    = afr_truncate_wind;
-	local->transaction.done   = afr_truncate_done;
-	local->transaction.unwind = afr_truncate_unwind;
+int
+afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t        *local       = NULL;
+        afr_private_t      *priv        = NULL;
+        int                 call_count  = -1;
+        int                 i           = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fsetxattr,
+                                           local->fd,
+                                           local->cont.fsetxattr.dict,
+                                           local->cont.fsetxattr.flags,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
 
-	loc_copy (&local->loc, loc);
 
-	local->transaction.main_frame = frame;
-	local->transaction.start   = 0;
-	local->transaction.len     = offset;
-	local->transaction.pending = AFR_DATA_PENDING;
+int
+afr_fsetxattr_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local   = frame->local;
 
-	afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        local->transaction.unwind (frame, this);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+        AFR_STACK_DESTROY (frame);
 
-	return 0;
+        return 0;
 }
 
+int
+afr_fsetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        afr_private_t    *priv              = NULL;
+        afr_local_t      *local             = NULL;
+        call_frame_t     *transaction_frame = NULL;
+        int               ret               = -1;
+        int               op_errno          = EINVAL;
 
-/* }}} */
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-/* {{{ ftruncate */
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
+                                   op_errno, out);
 
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
+                                   op_errno, out);
 
-int
-afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
+        priv = this->private;
 
-	local = frame->local;
-	priv  = this->private;
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
+        QUORUM_CHECK(fsetxattr,out);
 
-	if (main_frame) {
-		local->cont.ftruncate.buf.st_ino = local->cont.ftruncate.ino;
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  &local->cont.ftruncate.buf);
-	}
-	return 0;
-}
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-int
-afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-			int32_t op_ret, int32_t op_errno, struct stat *buf)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
 
-	int child_index = (long) cookie;
-	int call_count  = -1;
-	int need_unwind = 0;
+        transaction_frame->local = local;
 
-	local = frame->local;
-	priv  = this->private;
+        local->op_ret = -1;
 
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
+        local->cont.fsetxattr.dict  = dict_ref (dict);
+        local->cont.fsetxattr.flags = flags;
 
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-				local->cont.ftruncate.buf = *buf;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        local->transaction.fop    = afr_fsetxattr_wind;
+        local->transaction.done   = afr_fsetxattr_done;
+        local->transaction.unwind = afr_fsetxattr_unwind;
 
-	if (need_unwind)
-		local->transaction.unwind (frame, this);
+        local->fd                 = fd_ref (fd);
 
-	call_count = afr_frame_return (frame);
+        local->transaction.main_frame = frame;
+        local->transaction.start  = LLONG_MAX - 1;
+        local->transaction.len    = 0;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+        }
+
+        return 0;
 }
 
+/* }}} */
 
-int
-afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	
-	int call_count = -1;
-	int i = 0;
 
-	local = frame->local;
-	priv = this->private;
+/* {{{ removexattr */
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
+int
+afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (removexattr, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  NULL);
+        }
+        return 0;
+}
 
-	local->call_count = call_count;
 
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->ftruncate,
-					   local->fd, local->cont.ftruncate.offset);
+int
+afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        int           call_count  = -1;
+        int           need_unwind = 0;
+        int           child_index = (long) cookie;
+
+        local = frame->local;
+        priv = this->private;
+
+        LOCK (&frame->lock);
+        {
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
+                if (local->success_count == priv->wait_count) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
 
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+
+int32_t
+afr_removexattr_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->removexattr,
+                                           &local->loc,
+                                           local->cont.removexattr.name,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
 }
 
 
 int
-afr_ftruncate_done (call_frame_t *frame, xlator_t *this)
+afr_removexattr_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-
-	local = frame->local;
+        afr_local_t * local = frame->local;
 
-	local->transaction.unwind (frame, this);
+        local->transaction.unwind (frame, this);
 
-	AFR_STACK_DESTROY (frame);
+        AFR_STACK_DESTROY (frame);
 
-	return 0;
+        return 0;
 }
 
 
 int
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset)
+afr_removexattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, const char *name, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
+        afr_private_t   *priv              = NULL;
+        afr_local_t     *local             = NULL;
+        call_frame_t    *transaction_frame = NULL;
+        int              ret               = -1;
+        int              op_errno          = 0;
 
-	int ret = -1;
+        VALIDATE_OR_GOTO (this, out);
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
+                                 name, op_errno, out);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
+                                 name, op_errno, out);
 
-	priv = this->private;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this->private, out);
+        VALIDATE_OR_GOTO (loc, out);
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
+        priv = this->private;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        QUORUM_CHECK(removexattr,out);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-	transaction_frame->local = local;
+        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
+        local = transaction_frame->local;
 
-	local->op = GF_FOP_FTRUNCATE;
-	local->op_ret = -1;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-	local->cont.ftruncate.offset  = offset;
-	local->cont.ftruncate.ino     = fd->inode->ino;
+        local->cont.removexattr.name = gf_strdup (name);
 
-	local->transaction.fop    = afr_ftruncate_wind;
-	local->transaction.done   = afr_ftruncate_done;
-	local->transaction.unwind = afr_ftruncate_unwind;
+        local->transaction.fop    = afr_removexattr_wind;
+        local->transaction.done   = afr_removexattr_done;
+        local->transaction.unwind = afr_removexattr_unwind;
 
-	local->fd = fd_ref (fd);
+        loc_copy (&local->loc, loc);
 
-	local->transaction.main_frame = frame;
-	local->transaction.start   = 0;
-	local->transaction.len     = offset;
-	local->transaction.pending = AFR_DATA_PENDING;
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
 
-	afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
-	op_ret = 0;
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+        }
 
-	return 0;
+        return 0;
 }
 
-/* }}} */
-
-/* {{{ utimens */
-
-
+/* ffremovexattr */
 int
-afr_utimens_unwind (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-	priv  = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		local->cont.utimens.buf.st_ino = local->cont.utimens.ino;
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno,
-				  &local->cont.utimens.buf);
-	}
-	return 0;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (fremovexattr, main_frame,
+                                  local->op_ret, local->op_errno,
+                                  NULL);
+        }
+        return 0;
 }
 
 
 int
-afr_utimens_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		      int32_t op_ret, int32_t op_errno, struct stat *buf)
+afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_local_t *   local       = NULL;
+        afr_private_t * priv        = NULL;
+        int             call_count  = -1;
+        int             need_unwind = 0;
+        int             child_index = (long) cookie;
 
-	int child_index = (long) cookie;
-	int call_count  = -1;
-	int need_unwind = 1;
+        local = frame->local;
+        priv = this->private;
 
-	local = frame->local;
-	priv = this->private;
+        LOCK (&frame->lock);
+        {
+                __inode_write_fop_cbk (frame, child_index, -1, this,
+                                       &op_ret, &op_errno, NULL, NULL,
+                                       xdata);
 
-	LOCK (&frame->lock);
-	{
-		if (child_went_down (op_ret, op_errno))
-			afr_transaction_child_died (frame, this, child_index);
+                if (local->success_count == priv->wait_count) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
 
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-				local->cont.utimens.buf = *buf;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        call_count = afr_frame_return (frame);
 
-	if (need_unwind)
-		local->transaction.unwind (frame, this);
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
 
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return 0;
 }
 
 
-int
-afr_utimens_wind (call_frame_t *frame, xlator_t *this)
+int32_t
+afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fremovexattr,
+                                           local->fd,
+                                           local->cont.removexattr.name,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
+int
+afr_fremovexattr_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t * local = frame->local;
 
-	local->call_count = call_count;
+        local->transaction.unwind (frame, this);
 
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_utimens_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->utimens,
-					   &local->loc, 
-					   local->cont.utimens.tv); 
+        AFR_STACK_DESTROY (frame);
 
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        return 0;
 }
 
 
 int
-afr_utimens_done (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr (call_frame_t *frame, xlator_t *this,
+                  fd_t *fd, const char *name, dict_t *xdata)
 {
-	afr_local_t * local = NULL;
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_ret   = -1;
+        int op_errno = 0;
 
-	local = frame->local;
+        VALIDATE_OR_GOTO (this, out);
 
-	local->transaction.unwind (frame, this);
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
+                                 name, op_errno, out);
 
-	AFR_STACK_DESTROY (frame);
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
+                                 name, op_errno, out);
 
-	return 0;
-}
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
+        priv = this->private;
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
 
-int
-afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2])
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
+        QUORUM_CHECK(fremovexattr, out);
 
-	int ret = -1;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
 
-	priv = this->private;
+        transaction_frame->local = local;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
+        local->op_ret = -1;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        local->cont.removexattr.name = gf_strdup (name);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        local->transaction.fop    = afr_fremovexattr_wind;
+        local->transaction.done   = afr_fremovexattr_done;
+        local->transaction.unwind = afr_fremovexattr_unwind;
+
+        local->fd = fd_ref (fd);
+
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
+
+        op_ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
+        }
 
-	transaction_frame->local = local;
+        return 0;
+}
 
-	local->op_ret = -1;
+static int
+afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
 
-	local->cont.utimens.tv[0] = tv[0];
-	local->cont.utimens.tv[1] = tv[1];
+static int
+afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int need_unwind = 0;
+        int read_child  = 0;
 
-	local->cont.utimens.ino  = loc->inode->ino;
+        local = frame->local;
+        priv  = this->private;
 
-	local->transaction.fop    = afr_utimens_wind;
-	local->transaction.done   = afr_utimens_done;
-	local->transaction.unwind = afr_utimens_unwind;
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
 
-	loc_copy (&local->loc, loc);
-	
-	local->transaction.main_frame = frame;
-	local->transaction.start   = 0;
-	local->transaction.len     = 0;
-	local->transaction.pending = AFR_METADATA_PENDING;
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
 
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
 
-	return 0;
-}
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
 
-/* }}} */
+        call_count = afr_frame_return (frame);
 
-/* {{{ setxattr */
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
 
+        return 0;
+}
 
-int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
+static int
+afr_fallocate_wind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fallocate,
+                                           local->fd,
+                                           local->cont.fallocate.mode,
+                                           local->cont.fallocate.offset,
+                                           local->cont.fallocate.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
 
-	local = frame->local;
-	priv  = this->private;
+static int
+afr_fallocate_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
+        local = frame->local;
 
-	if (main_frame) {
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno)
-	}
-	return 0;
-}
+        local->transaction.unwind (frame, this);
 
+        AFR_STACK_DESTROY (frame);
 
-int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		       int32_t op_ret, int32_t op_errno)
+        return 0;
+}
+
+static int
+afr_do_fallocate (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        call_frame_t * transaction_frame = NULL;
+        afr_local_t *  local             = NULL;
+        int op_ret   = -1;
+        int op_errno = 0;
 
-	int call_count  = -1;
-	int need_unwind = 0;
+        local = frame->local;
 
-	local = frame->local;
-	priv = this->private;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
+        transaction_frame->local = local;
+        frame->local = NULL;
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        local->op = GF_FOP_FALLOCATE;
 
-	if (need_unwind)
-		local->transaction.unwind (frame, this);
+        local->transaction.fop    = afr_fallocate_wind;
+        local->transaction.done   = afr_fallocate_done;
+        local->transaction.unwind = afr_fallocate_unwind;
 
-	call_count = afr_frame_return (frame);
+        local->transaction.main_frame = frame;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
-}
+        local->transaction.start   = local->cont.fallocate.offset;
+        local->transaction.len     = 0;
 
+        /* fallocate can modify the file size */
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fallocate, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
 
 int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
 
-	int call_count = -1;
-	int i = 0;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	local = frame->local;
-	priv = this->private;
+        priv = this->private;
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(fallocate,out);
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
 
-	local->call_count = call_count;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->setxattr,
-					   &local->loc, 
-					   local->cont.setxattr.dict,
-					   local->cont.setxattr.flags); 
+        local->cont.fallocate.mode = mode;
+        local->cont.fallocate.offset  = offset;
+        local->cont.fallocate.len = len;
 
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
-}
+        local->fd = fd_ref (fd);
 
+        afr_open_fd_fix (fd, this);
 
-int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t * local = frame->local;
+        afr_do_fallocate (frame, this);
 
-	local->transaction.unwind (frame, this);
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        }
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+        return 0;
 }
 
+/* }}} */
 
-int
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, dict_t *dict, int32_t flags)
+/* {{{ discard */
+
+static int
+afr_discard_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame)
+                        main_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (discard, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.inode_wfop.prebuf,
+                                  &local->cont.inode_wfop.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
 
-	int ret = -1;
+static int
+afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int need_unwind = 0;
+        int read_child  = 0;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        local = frame->local;
+        priv  = this->private;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
 
-	priv = this->private;
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
+                __inode_write_fop_cbk (frame, child_index, read_child, this,
+                                       &op_ret, &op_errno, prebuf, postbuf,
+                                       xdata);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+                if ((local->success_count >= priv->wait_count)
+                    && local->read_child_returned) {
+                        need_unwind = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        if (need_unwind)
+                local->transaction.unwind (frame, this);
 
-	transaction_frame->local = local;
+        call_count = afr_frame_return (frame);
 
-	local->op_ret = -1;
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
 
-	local->cont.setxattr.dict  = dict_ref (dict);
-	local->cont.setxattr.flags = flags;
+        return 0;
+}
 
-	local->transaction.fop    = afr_setxattr_wind;
-	local->transaction.done   = afr_setxattr_done;
-	local->transaction.unwind = afr_setxattr_unwind;
+static int
+afr_discard_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_discard_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->discard,
+                                           local->fd,
+                                           local->cont.discard.offset,
+                                           local->cont.discard.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
 
-	loc_copy (&local->loc, loc);
+static int
+afr_discard_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
 
-	local->transaction.main_frame = frame;
-	local->transaction.start   = 0;
-	local->transaction.len     = 0;
-	local->transaction.pending = AFR_METADATA_PENDING;
+        local = frame->local;
 
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        local->transaction.unwind (frame, this);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
+        AFR_STACK_DESTROY (frame);
 
-	return 0;
+        return 0;
 }
 
-/* }}} */
+static int
+afr_do_discard (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t * transaction_frame = NULL;
+        afr_local_t *  local             = NULL;
+        int op_ret   = -1;
+        int op_errno = 0;
 
-/* {{{ removexattr */
+        local = frame->local;
 
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
 
-int
-afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
+        transaction_frame->local = local;
+        frame->local = NULL;
 
-	local = frame->local;
-	priv  = this->private;
+        local->op = GF_FOP_DISCARD;
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
+        local->transaction.fop    = afr_discard_wind;
+        local->transaction.done   = afr_discard_done;
+        local->transaction.unwind = afr_discard_unwind;
 
-	if (main_frame) {
-		AFR_STACK_UNWIND (main_frame, local->op_ret, local->op_errno)
-	}
-	return 0;
-}
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = 0;
 
+        op_ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+            op_errno = -op_ret;
+            goto out;
+        }
+
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (discard, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
 
 int
-afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-			  int32_t op_ret, int32_t op_errno)
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        call_frame_t   *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = 0;
 
-	int call_count  = -1;
-	int need_unwind = 0;
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	local = frame->local;
-	priv = this->private;
+        priv = this->private;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->wait_count) {
-				need_unwind = 1;
-			}
-		}
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(discard, out);
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
 
-	if (need_unwind)
-		local->transaction.unwind (frame, this);
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
 
-	call_count = afr_frame_return (frame);
+        local->cont.discard.offset  = offset;
+        local->cont.discard.len = len;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        local->fd = fd_ref (fd);
+
+        afr_open_fd_fix (fd, this);
+
+        afr_do_discard(frame, this);
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (transaction_frame)
+                        AFR_STACK_DESTROY (transaction_frame);
+                AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        }
+
+        return 0;
 }
 
 
-int32_t
-afr_removexattr_wind (call_frame_t *frame, xlator_t *this)
+/* {{{ zerofill */
+
+static int
+afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
+        afr_local_t     *local            = NULL;
+        call_frame_t    *main_frame       = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (local->transaction.main_frame) {
+                        main_frame = local->transaction.main_frame;
+                }
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
+
+        if (main_frame) {
+                AFR_STACK_UNWIND (zerofill, main_frame, local->op_ret,
+                                  local->op_errno,
+                                  &local->cont.zerofill.prebuf,
+                                  &local->cont.zerofill.postbuf,
+                                  NULL);
+        }
+        return 0;
+}
 
-	int call_count = -1;
-	int i = 0;
+static int
+afr_zerofill_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t       *local             = NULL;
+        afr_private_t     *priv              = NULL;
+        int                child_index       = (long) cookie;
+        int                call_count        = -1;
+        int                need_unwind       = 0;
+        int                read_child        = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+
+        LOCK (&frame->lock);
+        {
+                if (child_index == read_child) {
+                        local->read_child_returned = _gf_true;
+                }
+
+                if (afr_fop_failed (op_ret, op_errno)) {
+                        afr_transaction_fop_failed (frame, this, child_index);
+                }
+
+                if (op_ret != -1) {
+                        if (local->success_count == 0) {
+                                local->op_ret = op_ret;
+                                local->cont.zerofill.prebuf  = *prebuf;
+                                local->cont.zerofill.postbuf = *postbuf;
+                        }
+
+                        if (child_index == read_child) {
+                                local->cont.zerofill.prebuf  = *prebuf;
+                                local->cont.zerofill.postbuf = *postbuf;
+                        }
+
+                        local->success_count++;
+
+                        if ((local->success_count >= priv->wait_count)
+                            && local->read_child_returned) {
+                                need_unwind = 1;
+                        }
+                }
+                local->op_errno = op_errno;
+        }
+        UNLOCK (&frame->lock);
+
+        if (need_unwind) {
+                local->transaction.unwind (frame, this);
+        }
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+        }
+
+        return 0;
+}
 
-	local = frame->local;
-	priv = this->private;
+static int
+afr_zerofill_wind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t    *local         = NULL;
+        afr_private_t  *priv          = NULL;
+        int             call_count    = -1;
+        int             i             = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+                        STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->zerofill,
+                                           local->fd,
+                                           local->cont.zerofill.offset,
+                                           local->cont.zerofill.len,
+                                           NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        return 0;
+}
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
+static int
+afr_zerofill_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
+        local = frame->local;
 
-	local->call_count = call_count;
+        local->transaction.unwind (frame, this);
 
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk, 
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->removexattr,
-					   &local->loc, 
-					   local->cont.removexattr.name);
+        AFR_STACK_DESTROY (frame);
 
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        return 0;
 }
 
-
-int
-afr_removexattr_done (call_frame_t *frame, xlator_t *this)
+static int
+afr_do_zerofill(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t * local = frame->local;
+        call_frame_t  *transaction_frame = NULL;
+        afr_local_t   *local             = NULL;
+        int            op_ret            = -1;
+        int            op_errno          = 0;
 
-	local->transaction.unwind (frame, this);
+        local = frame->local;
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
-}
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame) {
+                goto out;
+        }
 
+        transaction_frame->local = local;
+        frame->local = NULL;
 
-int
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
+        local->op = GF_FOP_ZEROFILL;
 
-	int ret = -1;
+        local->transaction.fop    = afr_zerofill_wind;
+        local->transaction.done   = afr_zerofill_done;
+        local->transaction.unwind = afr_zerofill_unwind;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        local->transaction.main_frame = frame;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-	VALIDATE_OR_GOTO (loc, out);
+        local->transaction.start   = local->cont.zerofill.offset;
+        local->transaction.len     = 0;
 
-	priv = this->private;
+        op_ret = afr_transaction (transaction_frame, this,
+                                  AFR_DATA_TRANSACTION);
+        if (op_ret < 0) {
+                op_errno = -op_ret;
+                goto out;
+        }
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
+        op_ret = 0;
+out:
+        if (op_ret < 0) {
+                if (transaction_frame) {
+                        AFR_STACK_DESTROY (transaction_frame);
+                }
+                AFR_STACK_UNWIND (zerofill, frame, op_ret, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
+}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+int
+afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              size_t len, dict_t *xdata)
+{
+        afr_private_t   *priv               = NULL;
+        afr_local_t     *local              = NULL;
+        call_frame_t    *transaction_frame  = NULL;
+        int              ret                = -1;
+        int              op_errno           = 0;
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	transaction_frame->local = local;
+        priv = this->private;
 
-	local->op_ret = -1;
+        if (afr_is_split_brain (this, fd->inode)) {
+                op_errno = EIO;
+                goto out;
+        }
+        QUORUM_CHECK(zerofill, out);
 
-	local->cont.removexattr.name = strdup (name);
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
 
-	local->transaction.fop    = afr_removexattr_wind;
-	local->transaction.done   = afr_removexattr_done;
-	local->transaction.unwind = afr_removexattr_unwind;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0) {
+                goto out;
+        }
+        local->cont.zerofill.offset  = offset;
+        local->cont.zerofill.len = len;
 
-	loc_copy (&local->loc, loc);
+        local->fd = fd_ref (fd);
 
-	local->transaction.main_frame = frame;
-	local->transaction.start   = 0;
-	local->transaction.len     = 0;
-	local->transaction.pending = AFR_METADATA_PENDING;
+        afr_open_fd_fix (fd, this);
 
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        afr_do_zerofill(frame, this);
 
-	op_ret = 0;
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-
-	return 0;
+        if (ret < 0) {
+                if (transaction_frame) {
+                        AFR_STACK_DESTROY (transaction_frame);
+                }
+                AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL,
+                                  NULL, NULL);
+        }
+
+        return 0;
 }
+
+/* }}} */
+
+
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 9c0b5cad3..8e93ca44a 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_WRITE_H__
@@ -22,42 +13,70 @@
 
 int32_t
 afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode);
+	   loc_t *loc, mode_t mode, dict_t *xdata);
 
 int32_t
 afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid);
+	   loc_t *loc, uid_t uid, gid_t gid, dict_t *xdata);
 
 int
 afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid);
+	    fd_t *fd, uid_t uid, gid_t gid, dict_t *xdata);
 
 int32_t
 afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode);
+	    fd_t *fd, mode_t mode, dict_t *xdata);
 
 int32_t
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, 
-	    struct iovec *vector, int32_t count, off_t offset);
+afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+	    struct iovec *vector, int32_t count, off_t offset,
+            uint32_t flags, struct iobref *iobref, dict_t *xdata);
 
 int32_t
 afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset);
+	      loc_t *loc, off_t offset, dict_t *xdata);
 
 int32_t
 afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset);
+	       fd_t *fd, off_t offset, dict_t *xdata);
 
 int32_t
 afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2]);
+	     loc_t *loc, struct timespec tv[2], dict_t *xdata);
+
+int
+afr_setattr (call_frame_t *frame, xlator_t *this,
+             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata);
+
+int
+afr_fsetattr (call_frame_t *frame, xlator_t *this,
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata);
 
 int32_t
 afr_setxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, dict_t *dict, int32_t flags);
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata);
+
+int32_t
+afr_fsetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
 
 int32_t
 afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name);
+		 loc_t *loc, const char *name, dict_t *xdata);
 
+int32_t
+afr_fremovexattr (call_frame_t *frame, xlator_t *this,
+                  fd_t *fd, const char *name, dict_t *xdata);
+
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
+
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata);
+
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
new file mode 100644
index 000000000..060d78f35
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -0,0 +1,2174 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "dict.h"
+#include "byte-order.h"
+#include "common-utils.h"
+
+#include "afr.h"
+#include "afr-transaction.h"
+
+#include <signal.h>
+
+
+#define LOCKED_NO       0x0        /* no lock held */
+#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path */
+#define LOCKED_LOWER    0x2        /* for lower path */
+
+#define AFR_TRACE_INODELK_IN(frame, this, params ...)           \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->inodelk_trace)                      \
+                        break;                                  \
+                afr_trace_inodelk_in (frame, this, params);     \
+        } while (0);
+
+#define AFR_TRACE_INODELK_OUT(frame, this, params ...)          \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->inodelk_trace)                      \
+                        break;                                  \
+                afr_trace_inodelk_out (frame, this, params);    \
+        } while (0);
+
+#define AFR_TRACE_ENTRYLK_IN(frame, this, params ...)           \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->entrylk_trace)                      \
+                        break;                                  \
+                afr_trace_entrylk_in (frame, this, params);     \
+        } while (0);
+
+#define AFR_TRACE_ENTRYLK_OUT(frame, this, params ...)          \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->entrylk_trace)                      \
+                        break;                                  \
+                afr_trace_entrylk_out (frame, this, params);    \
+        } while (0);
+
+int
+afr_entry_lockee_cmp (const void *l1, const void *l2)
+{
+        const afr_entry_lockee_t       *r1 = l1;
+        const afr_entry_lockee_t       *r2 = l2;
+        int                            ret = 0;
+        uuid_t                         gfid1 = {0};
+        uuid_t                         gfid2 = {0};
+
+        loc_gfid ((loc_t*)&r1->loc, gfid1);
+        loc_gfid ((loc_t*)&r2->loc, gfid2);
+        ret = uuid_compare (gfid1, gfid2);
+        /*Entrylks with NULL basename are the 'smallest'*/
+        if (ret == 0) {
+                if (!r1->basename)
+                        return -1;
+                if (!r2->basename)
+                        return 1;
+                ret = strcmp (r1->basename, r2->basename);
+        }
+
+        if (ret <= 0)
+                return -1;
+        else
+                return 1;
+}
+
+int afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+
+static int
+afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this);
+
+static uint64_t afr_lock_number = 1;
+
+static uint64_t
+get_afr_lock_number ()
+{
+        return (++afr_lock_number);
+}
+
+int
+afr_set_lock_number (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t         *local    = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        int_lock->lock_number = get_afr_lock_number ();
+
+        return 0;
+}
+
+void
+afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner)
+{
+        gf_log (this->name, GF_LOG_TRACE,
+                "Setting lk-owner=%llu",
+                (unsigned long long) (unsigned long)lk_owner);
+
+        set_lk_owner_from_ptr (&frame->root->lk_owner, lk_owner);
+}
+
+static int
+is_afr_lock_selfheal (afr_local_t *local)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        int                  ret      = -1;
+
+        int_lock = &local->internal_lock;
+
+        switch (int_lock->selfheal_lk_type) {
+        case AFR_DATA_SELF_HEAL_LK:
+        case AFR_METADATA_SELF_HEAL_LK:
+                ret = 1;
+                break;
+        case AFR_ENTRY_SELF_HEAL_LK:
+                ret = 0;
+                break;
+        }
+
+        return ret;
+
+}
+
+int32_t
+internal_lock_count (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int32_t call_count = 0;
+        int i = 0;
+
+        local = frame->local;
+        priv  = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i])
+                        ++call_count;
+        }
+
+        return call_count;
+}
+
+static void
+afr_print_inodelk (char *str, int size, int cmd,
+                   struct gf_flock *flock, gf_lkowner_t *owner)
+{
+        char *cmd_str = NULL;
+        char *type_str = NULL;
+
+        switch (cmd) {
+#if F_GETLK != F_GETLK64
+        case F_GETLK64:
+#endif
+        case F_GETLK:
+                cmd_str = "GETLK";
+                break;
+
+#if F_SETLK != F_SETLK64
+        case F_SETLK64:
+#endif
+        case F_SETLK:
+                cmd_str = "SETLK";
+                break;
+
+#if F_SETLKW != F_SETLKW64
+        case F_SETLKW64:
+#endif
+        case F_SETLKW:
+                cmd_str = "SETLKW";
+                break;
+
+        default:
+                cmd_str = "<null>";
+                break;
+        }
+
+        switch (flock->l_type) {
+        case F_RDLCK:
+                type_str = "READ";
+                break;
+        case F_WRLCK:
+                type_str = "WRITE";
+                break;
+        case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+        default:
+                type_str = "UNKNOWN";
+                break;
+        }
+
+        snprintf (str, size, "lock=INODELK, cmd=%s, type=%s, "
+                  "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
+                  cmd_str, type_str, (unsigned long long) flock->l_start,
+                  (unsigned long long) flock->l_len,
+                  (unsigned long long) flock->l_pid,
+                  lkowner_utoa (owner));
+
+}
+
+static void
+afr_print_lockee (char *str, int size, loc_t *loc, fd_t *fd,
+                  int child_index)
+{
+        snprintf (str, size, "path=%s, fd=%p, child=%d",
+                  loc->path ? loc->path : "<nul>",
+                  fd ? fd : NULL,
+                  child_index);
+}
+
+void
+afr_print_entrylk (char *str, int size, const char *basename,
+                   gf_lkowner_t *owner)
+{
+        snprintf (str, size, "Basename=%s, lk-owner=%s",
+                  basename ? basename : "<nul>",
+                  lkowner_utoa (owner));
+}
+
+static void
+afr_print_verdict (int op_ret, int op_errno, char *str)
+{
+        if (op_ret < 0) {
+                if (op_errno == EAGAIN)
+                        strcpy (str, "EAGAIN");
+                else
+                        strcpy (str, "FAILED");
+        }
+        else
+                strcpy (str, "GRANTED");
+}
+
+static void
+afr_set_lock_call_type (afr_lock_call_type_t lock_call_type,
+                        char *lock_call_type_str,
+                        afr_internal_lock_t *int_lock)
+{
+        switch (lock_call_type) {
+        case AFR_INODELK_TRANSACTION:
+                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
+                        strcpy (lock_call_type_str, "AFR_INODELK_TRANSACTION");
+                else
+                        strcpy (lock_call_type_str, "AFR_INODELK_SELFHEAL");
+                break;
+        case AFR_INODELK_NB_TRANSACTION:
+                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
+                        strcpy (lock_call_type_str, "AFR_INODELK_NB_TRANSACTION");
+                else
+                        strcpy (lock_call_type_str, "AFR_INODELK_NB_SELFHEAL");
+                break;
+        case AFR_ENTRYLK_TRANSACTION:
+                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
+                        strcpy (lock_call_type_str, "AFR_ENTRYLK_TRANSACTION");
+                else
+                        strcpy (lock_call_type_str, "AFR_ENTRYLK_SELFHEAL");
+                break;
+        case AFR_ENTRYLK_NB_TRANSACTION:
+                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
+                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_TRANSACTION");
+                else
+                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_SELFHEAL");
+                break;
+        default:
+                strcpy (lock_call_type_str, "UNKNOWN");
+                break;
+        }
+
+}
+
+static void
+afr_trace_inodelk_out (call_frame_t *frame, xlator_t *this,
+                       afr_lock_call_type_t lock_call_type,
+                       afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
+                       int op_ret, int op_errno, int32_t child_index)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+
+        char lockee[256];
+        char lock_call_type_str[256];
+        char verdict[16];
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+
+        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
+
+        afr_print_verdict (op_ret, op_errno, verdict);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[%s %s] [%s] lk-owner=%s Lockee={%s} Number={%llu}",
+                lock_call_type_str,
+                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
+                verdict, lkowner_utoa (&frame->root->lk_owner), lockee,
+                (unsigned long long) int_lock->lock_number);
+
+}
+
+static void
+afr_trace_inodelk_in (call_frame_t *frame, xlator_t *this,
+                      afr_lock_call_type_t lock_call_type,
+                      afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
+                      int32_t cmd, int32_t child_index)
+{
+        afr_local_t         *local    = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+
+        char lock[256];
+        char lockee[256];
+        char lock_call_type_str[256];
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        afr_print_inodelk (lock, 256, cmd, flock, &frame->root->lk_owner);
+        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+
+        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
+                lock_call_type_str,
+                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
+                lock, lockee,
+                (unsigned long long) int_lock->lock_number);
+
+}
+
+static void
+afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
+                      afr_lock_call_type_t lock_call_type,
+                      afr_lock_op_type_t lk_op_type, const char *basename,
+                      int32_t cookie)
+{
+        afr_local_t         *local    = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_private_t       *priv     = NULL;
+        int                 child_index = 0;
+        int                 lockee_no = 0;
+
+        char lock[256];
+        char lockee[256];
+        char lock_call_type_str[256];
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        priv     = this->private;
+
+        if (!priv->entrylk_trace) {
+                return;
+        }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
+
+        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
+
+        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}, Cookie={%d}",
+                lock_call_type_str,
+                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
+                lock, lockee,
+                (unsigned long long) int_lock->lock_number,
+                cookie);
+}
+
+static void
+afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
+                       afr_lock_call_type_t lock_call_type,
+                       afr_lock_op_type_t lk_op_type, const char *basename,
+                       int op_ret, int op_errno, int32_t cookie)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
+        int                 lockee_no = 0;
+        int                 child_index = 0;
+
+        char lock[256];
+        char lockee[256];
+        char lock_call_type_str[256];
+        char verdict[16];
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        priv     = this->private;
+
+        if (!priv->entrylk_trace) {
+                return;
+        }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
+
+        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
+
+        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
+
+        afr_print_verdict (op_ret, op_errno, verdict);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu} Cookie={%d}",
+                lock_call_type_str,
+                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
+                verdict,
+                lock, lockee,
+                (unsigned long long) int_lock->lock_number,
+                cookie);
+
+}
+
+static int
+transaction_lk_op (afr_local_t *local)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        int ret = -1;
+
+        int_lock = &local->internal_lock;
+
+        if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK) {
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "lk op is for a transaction");
+                ret = 1;
+        }
+        else if (int_lock->transaction_lk_type == AFR_SELFHEAL_LK) {
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "lk op is for a self heal");
+
+                ret = 0;
+        }
+
+        if (ret == -1)
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "lk op is not set");
+
+        return ret;
+
+}
+
+static int
+is_afr_lock_transaction (afr_local_t *local)
+{
+        int ret = 0;
+
+        switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+                ret = 1;
+                break;
+
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+                ret = 0;
+                break;
+
+        }
+
+        return ret;
+}
+
+int
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count)
+{
+        int     ret     = -1;
+
+        loc_copy (&lockee->loc, loc);
+        lockee->basename        = (basename)? gf_strdup (basename): NULL;
+        if (basename && !lockee->basename)
+                goto out;
+
+        lockee->locked_count    = 0;
+        lockee->locked_nodes    = GF_CALLOC (child_count,
+                                             sizeof (*lockee->locked_nodes),
+                                             gf_afr_mt_afr_node_character);
+
+        if (!lockee->locked_nodes)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock)
+{
+        int     i   = 0;
+
+        for (i = 0; i < int_lock->lockee_count; i++) {
+                loc_wipe (&int_lock->lockee[i].loc);
+                if (int_lock->lockee[i].basename)
+                        GF_FREE (int_lock->lockee[i].basename);
+                if (int_lock->lockee[i].locked_nodes)
+                       GF_FREE (int_lock->lockee[i].locked_nodes);
+        }
+
+        return;
+}
+
+static int
+initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t         *local    = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_private_t       *priv     = NULL;
+
+        int i = 0;
+
+        priv     = this->private;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        int_lock->entrylk_lock_count = 0;
+        int_lock->lock_op_ret        = -1;
+        int_lock->lock_op_errno      = 0;
+
+        for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+                if (!int_lock->lockee[i].locked_nodes)
+                        break;
+                int_lock->lockee[i].locked_count = 0;
+                memset (int_lock->lockee[i].locked_nodes, 0,
+                        sizeof (*int_lock->lockee[i].locked_nodes) *
+                        priv->child_count);
+        }
+
+        return 0;
+}
+
+static int
+initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t         *local    = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_private_t       *priv     = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
+
+        priv     = this->private;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        inodelk->lock_count    = 0;
+        int_lock->lk_attempted_count = 0;
+        int_lock->lock_op_ret   = -1;
+        int_lock->lock_op_errno = 0;
+
+        memset (inodelk->locked_nodes, 0,
+                sizeof (*inodelk->locked_nodes) * priv->child_count);
+        memset (int_lock->locked_nodes, 0,
+                sizeof (*int_lock->locked_nodes) * priv->child_count);
+
+        return 0;
+}
+
+loc_t *
+lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
+{
+        int ret = 0;
+
+        ret = uuid_compare (l1->inode->gfid, l2->inode->gfid);
+
+        if (ret == 0)
+                ret = strcmp (b1, b2);
+
+        if (ret <= 0)
+                return l1;
+        else
+                return l2;
+}
+
+int
+afr_lockee_locked_nodes_count (afr_internal_lock_t *int_lock)
+{
+        int call_count  = 0;
+        int i           = 0;
+
+        for (i = 0; i < int_lock->lockee_count; i++)
+                call_count += int_lock->lockee[i].locked_count;
+
+        return call_count;
+}
+
+int
+afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
+
+{
+        int i = 0;
+        int call_count = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (locked_nodes[i] & LOCKED_YES)
+                        call_count++;
+        }
+
+        return call_count;
+}
+
+/* FIXME: What if UNLOCK fails */
+static int32_t
+afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_local_t         *local    = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        int call_count = 0;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        LOCK (&frame->lock);
+        {
+                call_count = --int_lock->lk_call_count;
+        }
+        UNLOCK (&frame->lock);
+
+        if (call_count == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "All internal locks unlocked");
+                int_lock->lock_cbk (frame, this);
+        }
+
+        return 0;
+}
+
+static int32_t
+afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_local_t         *local = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk = NULL;
+        int32_t             child_index = (long)cookie;
+        afr_private_t       *priv = NULL;
+
+        local = frame->local;
+        int_lock = &local->internal_lock;
+
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
+                               AFR_UNLOCK_OP, NULL, op_ret,
+                               op_errno, child_index);
+
+        priv = this->private;
+
+        if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on subvolume %s "
+                        "with lock owner %s", local->loc.path,
+                        priv->children[child_index]->name,
+                        lkowner_utoa (&frame->root->lk_owner));
+        }
+
+
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        inodelk->locked_nodes[child_index] &= LOCKED_NO;
+        if (local->transaction.eager_lock)
+                local->transaction.eager_lock[child_index] = 0;
+
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+
+        return 0;
+
+}
+
+static int
+afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
+        afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
+        struct gf_flock flock = {0,};
+        struct gf_flock full_flock = {0,};
+        struct gf_flock *flock_use = NULL;
+        int call_count = 0;
+        int i = 0;
+        int piggyback = 0;
+        afr_fd_ctx_t        *fd_ctx      = NULL;
+
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        priv     = this->private;
+
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
+        flock.l_type  = F_UNLCK;
+
+        full_flock.l_type = F_UNLCK;
+        call_count = afr_locked_nodes_count (inodelk->locked_nodes,
+                                             priv->child_count);
+
+        int_lock->lk_call_count = call_count;
+
+        if (!call_count) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "No internal locks unlocked");
+                int_lock->lock_cbk (frame, this);
+                goto out;
+        }
+
+        if (local->fd)
+                fd_ctx = afr_fd_ctx_get (local->fd, this);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((inodelk->locked_nodes[i] & LOCKED_YES) != LOCKED_YES)
+                        continue;
+
+                if (local->fd) {
+                        flock_use = &flock;
+                        if (!local->transaction.eager_lock[i]) {
+                                goto wind;
+                        }
+
+                        piggyback = 0;
+
+                        LOCK (&local->fd->lock);
+                        {
+                                if (fd_ctx->lock_piggyback[i]) {
+                                        fd_ctx->lock_piggyback[i]--;
+                                        piggyback = 1;
+                                } else {
+                                        fd_ctx->lock_acquired[i]--;
+                                }
+                        }
+                        UNLOCK (&local->fd->lock);
+
+                        if (piggyback) {
+                                afr_unlock_inodelk_cbk (frame, (void *) (long) i,
+                                                        this, 1, 0, NULL);
+                                if (!--call_count)
+                                        break;
+                                continue;
+                        }
+
+                        flock_use = &full_flock;
+                wind:
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
+                                              AFR_UNLOCK_OP, flock_use, F_SETLK,
+                                              i);
+
+                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
+                                           (void *) (long)i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->finodelk,
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
+
+                        if (!--call_count)
+                                break;
+
+                } else {
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
+                                              AFR_UNLOCK_OP, &flock, F_SETLK, i);
+
+                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
+                                           (void *) (long)i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->inodelk,
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+out:
+        return 0;
+}
+
+static int32_t
+afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_local_t         *local = NULL;
+        afr_private_t       *priv  = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        int32_t             child_index = 0;
+        int                 lockee_no   = 0;
+
+        priv = this->private;
+        lockee_no = (int)((long) cookie) / priv->child_count;
+        child_index = (int) ((long) cookie) % priv->child_count;
+
+        local = frame->local;
+        int_lock = &local->internal_lock;
+
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
+                               AFR_UNLOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
+                               op_errno, (int) ((long)cookie));
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: unlock failed on %d, reason: %s",
+                        local->loc.path, child_index, strerror (op_errno));
+        }
+
+        int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO;
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
+
+        return 0;
+}
+
+static int
+afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     call_count      = 0;
+        int                     index           = 0;
+        int                     lockee_no       = 0;
+        int                     copies          = 0;
+        int                     i               = -1;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        priv     = this->private;
+        copies   = priv->child_count;
+
+        call_count = afr_lockee_locked_nodes_count (int_lock);
+
+        int_lock->lk_call_count = call_count;
+
+        if (!call_count){
+                gf_log (this->name, GF_LOG_TRACE,
+                        "No internal locks unlocked");
+                int_lock->lock_cbk (frame, this);
+                goto out;
+        }
+
+        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+                lockee_no = i / copies;
+                index     = i % copies;
+                if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) {
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                              AFR_UNLOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              i);
+
+                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
+                                           (void *) (long) i,
+                                           priv->children[index],
+                                           priv->children[index]->fops->entrylk,
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+out:
+        return 0;
+
+}
+
+static int32_t
+afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     cky             = (long) cookie;
+        int                     child_index     = 0;
+        int                     lockee_no       = 0;
+
+        priv     = this->private;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        child_index = ((int)cky) % priv->child_count;
+        lockee_no   = ((int)cky) / priv->child_count;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        if (op_errno == ENOSYS) {
+                                /* return ENOTSUP */
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "subvolume does not support locking. "
+                                        "please load features/locks xlator on server");
+                                local->op_ret = op_ret;
+                                int_lock->lock_op_ret = op_ret;
+                        }
+
+                        local->op_errno              = op_errno;
+                        int_lock->lock_op_errno      = op_errno;
+                }
+
+		int_lock->lk_attempted_count++;
+        }
+        UNLOCK (&frame->lock);
+
+        if ((op_ret == -1) &&
+            (op_errno == ENOSYS)) {
+                afr_unlock (frame, this);
+        } else {
+                if (op_ret == 0) {
+                        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+                            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+                                int_lock->lockee[lockee_no].locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lockee[lockee_no].locked_count++;
+                                int_lock->entrylk_lock_count++;
+                        } else {
+                                int_lock->locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lock_count++;
+                        }
+                }
+                afr_lock_blocking (frame, this, cky + 1);
+        }
+
+        return 0;
+}
+
+static int32_t
+afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
+                               AFR_LOCK_OP, NULL, op_ret,
+                               op_errno, (long) cookie);
+
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+        return 0;
+
+}
+
+static int32_t
+afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
+                               AFR_LOCK_OP, NULL, op_ret,
+                               op_errno, (long)cookie);
+
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+        return 0;
+}
+
+static int
+afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
+        afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
+
+        priv     = this->private;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                memcpy (inodelk->locked_nodes, int_lock->locked_nodes,
+                        sizeof (*inodelk->locked_nodes) * priv->child_count);
+                inodelk->lock_count = int_lock->lock_count;
+                break;
+
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+                /*entrylk_count is being used in both non-blocking and blocking
+                 * modes */
+                break;
+        }
+
+        return 0;
+
+}
+
+static inline gf_boolean_t
+afr_is_entrylk (afr_internal_lock_t *int_lock,
+                afr_transaction_type trans_type)
+{
+        gf_boolean_t is_entrylk = _gf_false;
+
+        if ((int_lock->transaction_lk_type == AFR_SELFHEAL_LK) &&
+            int_lock->selfheal_lk_type == AFR_ENTRY_SELF_HEAL_LK) {
+
+                is_entrylk = _gf_true;
+
+        } else if ((int_lock->transaction_lk_type == AFR_TRANSACTION_LK) &&
+                 (trans_type == AFR_ENTRY_TRANSACTION ||
+                  trans_type == AFR_ENTRY_RENAME_TRANSACTION)) {
+
+                is_entrylk = _gf_true;
+
+        } else {
+                is_entrylk = _gf_false;
+        }
+
+        return is_entrylk;
+}
+
+static gf_boolean_t
+_is_lock_wind_needed (afr_local_t *local, int child_index)
+{
+        if (!local->child_up[child_index])
+                return _gf_false;
+
+        return _gf_true;
+}
+
+int
+afr_lock_blocking (call_frame_t *frame, xlator_t *this, int cookie)
+{
+        afr_internal_lock_t *int_lock    = NULL;
+        afr_inodelk_t       *inodelk     = NULL;
+        afr_local_t         *local       = NULL;
+        afr_private_t       *priv        = NULL;
+        struct gf_flock flock = {0,};
+        uint64_t ctx = 0;
+        int ret = 0;
+        int child_index = 0;
+        int lockee_no   = 0;
+        gf_boolean_t is_entrylk = _gf_false;
+
+        local         = frame->local;
+        int_lock      = &local->internal_lock;
+        priv          = this->private;
+        child_index   = cookie % priv->child_count;
+        lockee_no     = cookie / priv->child_count;
+        is_entrylk    = afr_is_entrylk (int_lock, local->transaction.type);
+
+
+        if (!is_entrylk) {
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                flock.l_start = inodelk->flock.l_start;
+                flock.l_len   = inodelk->flock.l_len;
+                flock.l_type  = inodelk->flock.l_type;
+        }
+
+        if (local->fd) {
+                ret = fd_ctx_get (local->fd, this, &ctx);
+
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "unable to get fd ctx for fd=%p",
+                                local->fd);
+
+                        local->op_ret           = -1;
+                        int_lock->lock_op_ret   = -1;
+
+                        afr_copy_locked_nodes (frame, this);
+
+                        afr_unlock (frame, this);
+
+                        return 0;
+                }
+        }
+
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+                if ((is_entrylk && int_lock->entrylk_lock_count == 0) ||
+                    (!is_entrylk && int_lock->lock_count == 0)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "unable to lock on even one child");
+
+                        local->op_ret           = -1;
+                        int_lock->lock_op_ret   = -1;
+
+                        afr_copy_locked_nodes (frame, this);
+
+                        afr_unlock(frame, this);
+
+                        return 0;
+                }
+        }
+
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+                /* we're done locking */
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "we're done locking");
+
+                afr_copy_locked_nodes (frame, this);
+
+                int_lock->lock_op_ret = 0;
+                int_lock->lock_cbk (frame, this);
+                return 0;
+        }
+
+        if (!_is_lock_wind_needed (local, child_index)) {
+                afr_lock_blocking (frame, this, cookie + 1);
+                return 0;
+        }
+
+        switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+
+                if (local->fd) {
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
+                                              AFR_LOCK_OP, &flock, F_SETLKW,
+                                              child_index);
+
+                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
+                                           (void *) (long) child_index,
+                                           priv->children[child_index],
+                                           priv->children[child_index]->fops->finodelk,
+                                           int_lock->domain, local->fd,
+                                           F_SETLKW, &flock, NULL);
+
+                } else {
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
+                                              AFR_LOCK_OP, &flock, F_SETLKW,
+                                              child_index);
+
+                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
+                                           (void *) (long) child_index,
+                                           priv->children[child_index],
+                                           priv->children[child_index]->fops->inodelk,
+                                           int_lock->domain, &local->loc,
+                                           F_SETLKW, &flock, NULL);
+                }
+
+                break;
+
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+                /*Accounting for child_index increments on 'down'
+                 *and 'fd-less' children */
+
+                if (local->fd) {
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
+                                              AFR_LOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              cookie);
+
+                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
+                                           (void *) (long) cookie,
+                                           priv->children[child_index],
+                                           priv->children[child_index]->fops->fentrylk,
+                                           int_lock->domain, local->fd,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+                } else {
+                        AFR_TRACE_ENTRYLK_IN (frame, this,
+                                              AFR_ENTRYLK_TRANSACTION,
+                                              AFR_LOCK_OP, local->transaction.basename,
+                                              child_index);
+
+                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
+                                           (void *) (long) cookie,
+                                           priv->children[child_index],
+                                           priv->children[child_index]->fops->entrylk,
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+                }
+
+                break;
+        }
+
+        return 0;
+}
+
+int32_t
+afr_blocking_lock (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
+        int                  up_count = 0;
+
+        priv     = this->private;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+                initialize_inodelk_variables (frame, this);
+                break;
+
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+                up_count = afr_up_children_count (local->child_up,
+                                                  priv->child_count);
+                int_lock->lk_call_count = int_lock->lk_expected_count
+                                        = (int_lock->lockee_count *
+                                           up_count);
+                initialize_entrylk_variables (frame, this);
+                break;
+        }
+
+        afr_lock_blocking (frame, this, 0);
+
+        return 0;
+}
+
+static int32_t
+afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        int call_count          = 0;
+        int child_index         = (long) cookie;
+        int copies              = 0;
+        int index               = 0;
+        int lockee_no           = 0;
+        afr_private_t       *priv = NULL;
+
+        priv = this->private;
+
+        copies = priv->child_count;
+        index = child_index % copies;
+        lockee_no = child_index / copies;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
+                               AFR_LOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
+                               op_errno, (long) cookie);
+
+	LOCK (&frame->lock);
+	{
+		if (op_ret < 0 ) {
+			if (op_errno == ENOSYS) {
+                        /* return ENOTSUP */
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on server");
+				local->op_ret         = op_ret;
+				int_lock->lock_op_ret = op_ret;
+
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+		} else if (op_ret == 0) {
+			int_lock->lockee[lockee_no].locked_nodes[index] |= \
+				LOCKED_YES;
+			int_lock->lockee[lockee_no].locked_count++;
+			int_lock->entrylk_lock_count++;
+		}
+
+                call_count = --int_lock->lk_call_count;
+        }
+        UNLOCK (&frame->lock);
+
+        if (call_count == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Last locking reply received");
+                /* all locks successful. Proceed to call FOP */
+                if (int_lock->entrylk_lock_count ==
+                                int_lock->lk_expected_count) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "All servers locked. Calling the cbk");
+                        int_lock->lock_op_ret = 0;
+                        int_lock->lock_cbk (frame, this);
+                }
+                /* Not all locks were successful. Unlock and try locking
+                   again, this time with serially blocking locks */
+                else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "%d servers locked. Trying again with blocking calls",
+                                int_lock->lock_count);
+
+                        afr_unlock(frame, this);
+                }
+        }
+
+        return 0;
+}
+
+int
+afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock   = NULL;
+        afr_local_t         *local      = NULL;
+        afr_private_t       *priv       = NULL;
+        afr_fd_ctx_t        *fd_ctx     = NULL;
+        int                 copies      = 0;
+        int                 index       = 0;
+        int                 lockee_no   = 0;
+        int32_t             call_count  = 0;
+        int i = 0;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        priv     = this->private;
+
+        copies = priv->child_count;
+        initialize_entrylk_variables (frame, this);
+
+        if (local->fd) {
+                fd_ctx = afr_fd_ctx_get (local->fd, this);
+                if (!fd_ctx) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "unable to get fd ctx for fd=%p",
+                                local->fd);
+
+                        local->op_ret           = -1;
+                        int_lock->lock_op_ret   = -1;
+                        local->op_errno         = EINVAL;
+                        int_lock->lock_op_errno = EINVAL;
+
+			afr_unlock (frame, this);
+                        return -1;
+                }
+
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
+                int_lock->lk_call_count = call_count;
+                int_lock->lk_expected_count = call_count;
+
+                if (!call_count) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "fd not open on any subvolumes. aborting.");
+                        afr_unlock (frame, this);
+                        goto out;
+                }
+
+                /* Send non-blocking entrylk calls only on up children
+                   and where the fd has been opened */
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
+
+                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->fentrylk,
+                                                   this->name, local->fd,
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
+                                if (!--call_count)
+                                        break;
+                        }
+                }
+        } else {
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
+                int_lock->lk_call_count = call_count;
+                int_lock->lk_expected_count = call_count;
+
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
+
+                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->entrylk,
+                                                   this->name, &int_lock->lockee[lockee_no].loc,
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
+
+                                if (!--call_count)
+                                        break;
+                        }
+                }
+        }
+out:
+        return 0;
+}
+
+int32_t
+afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
+        afr_local_t         *local    = NULL;
+        int call_count  = 0;
+        int child_index = (long) cookie;
+        afr_fd_ctx_t        *fd_ctx = NULL;
+
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_NB_TRANSACTION,
+                               AFR_LOCK_OP, NULL, op_ret,
+                               op_errno, (long) cookie);
+
+	if (local->fd)
+		fd_ctx = afr_fd_ctx_get (local->fd, this);
+
+        LOCK (&frame->lock);
+        {
+		if (op_ret < 0) {
+			if (op_errno == ENOSYS) {
+				/* return ENOTSUP */
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on "
+					"server");
+				local->op_ret                = op_ret;
+				int_lock->lock_op_ret        = op_ret;
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+			if (local->transaction.eager_lock)
+				local->transaction.eager_lock[child_index] = 0;
+		} else {
+			inodelk->locked_nodes[child_index] |= LOCKED_YES;
+			inodelk->lock_count++;
+
+			if (local->transaction.eager_lock &&
+			    local->transaction.eager_lock[child_index] &&
+			    local->fd) {
+				/* piggybacked */
+				if (op_ret == 1) {
+					/* piggybacked */
+				} else if (op_ret == 0) {
+					/* lock acquired from server */
+                                        fd_ctx->lock_acquired[child_index]++;
+				}
+			}
+		}
+
+                call_count = --int_lock->lk_call_count;
+        }
+        UNLOCK (&frame->lock);
+
+        if (call_count == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Last inode locking reply received");
+                /* all locks successful. Proceed to call FOP */
+                if (inodelk->lock_count == int_lock->lk_expected_count) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "All servers locked. Calling the cbk");
+                        int_lock->lock_op_ret = 0;
+                        int_lock->lock_cbk (frame, this);
+                }
+                /* Not all locks were successful. Unlock and try locking
+                   again, this time with serially blocking locks */
+                else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "%d servers locked. Trying again with blocking calls",
+                                int_lock->lock_count);
+
+                        afr_unlock(frame, this);
+                }
+        }
+
+        return 0;
+}
+
+int
+afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
+        afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
+        afr_fd_ctx_t        *fd_ctx   = NULL;
+        int32_t             call_count = 0;
+        int                 i          = 0;
+        int                 ret        = 0;
+        struct              gf_flock flock = {0,};
+        struct              gf_flock full_flock = {0,};
+        struct              gf_flock *flock_use = NULL;
+        int                 piggyback = 0;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        priv     = this->private;
+
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
+        flock.l_type  = inodelk->flock.l_type;
+
+        full_flock.l_type = inodelk->flock.l_type;
+
+        initialize_inodelk_variables (frame, this);
+
+        if (local->fd) {
+                fd_ctx = afr_fd_ctx_get (local->fd, this);
+                if (!fd_ctx) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "unable to get fd ctx for fd=%p",
+                                local->fd);
+
+                        local->op_ret           = -1;
+                        int_lock->lock_op_ret   = -1;
+                        local->op_errno         = EINVAL;
+                        int_lock->lock_op_errno = EINVAL;
+
+			afr_unlock (frame, this);
+                        ret = -1;
+                        goto out;
+                }
+
+                call_count = internal_lock_count (frame, this);
+                int_lock->lk_call_count = call_count;
+                int_lock->lk_expected_count = call_count;
+
+                if (!call_count) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "fd not open on any subvolumes. aborting.");
+                        afr_unlock (frame, this);
+                        goto out;
+                }
+
+                /* Send non-blocking inodelk calls only on up children
+                   and where the fd has been opened */
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!local->child_up[i])
+                                continue;
+
+                        flock_use = &flock;
+                        if (!local->transaction.eager_lock_on) {
+                                goto wind;
+                        }
+
+                        piggyback = 0;
+                        local->transaction.eager_lock[i] = 1;
+
+			afr_set_delayed_post_op (frame, this);
+
+                        LOCK (&local->fd->lock);
+                        {
+                                if (fd_ctx->lock_acquired[i]) {
+                                        fd_ctx->lock_piggyback[i]++;
+                                        piggyback = 1;
+                                }
+                        }
+                        UNLOCK (&local->fd->lock);
+
+                        if (piggyback) {
+                                /* (op_ret == 1) => indicate piggybacked lock */
+                                afr_nonblocking_inodelk_cbk (frame, (void *) (long) i,
+                                                             this, 1, 0, NULL);
+                                if (!--call_count)
+                                        break;
+                                continue;
+                        }
+                        flock_use = &full_flock;
+                wind:
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_NB_TRANSACTION,
+                                              AFR_LOCK_OP, flock_use, F_SETLK, i);
+
+                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->finodelk,
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        } else {
+                call_count = internal_lock_count (frame, this);
+                int_lock->lk_call_count = call_count;
+                int_lock->lk_expected_count = call_count;
+
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!local->child_up[i])
+                                continue;
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_NB_TRANSACTION,
+                                              AFR_LOCK_OP, &flock, F_SETLK, i);
+
+                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->inodelk,
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+out:
+        return ret;
+}
+
+int32_t
+afr_unlock (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (transaction_lk_op (local)) {
+                if (is_afr_lock_transaction (local))
+                        afr_unlock_inodelk (frame, this);
+                else
+                        afr_unlock_entrylk (frame, this);
+
+        } else {
+                if (is_afr_lock_selfheal (local))
+                        afr_unlock_inodelk (frame, this);
+                else
+                        afr_unlock_entrylk (frame, this);
+        }
+
+        return 0;
+}
+
+int
+afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
+                       unsigned char *locked_nodes)
+{
+        afr_private_t *priv  = NULL;
+        afr_fd_ctx_t  *fdctx = NULL;
+        uint64_t       tmp   = 0;
+        int            ret   = 0;
+
+        priv = this->private;
+
+        ret = afr_fd_ctx_set (this, fd);
+        if (ret)
+                goto out;
+
+        ret = fd_ctx_get (fd, this, &tmp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "failed to get the fd ctx");
+                goto out;
+        }
+        fdctx = (afr_fd_ctx_t *) (long) tmp;
+
+        GF_ASSERT (fdctx->locked_on);
+
+        memcpy (fdctx->locked_on, locked_nodes,
+                priv->child_count);
+
+out:
+        return ret;
+}
+
+static int
+__is_fd_saved (xlator_t *this, fd_t *fd)
+{
+        afr_locked_fd_t *locked_fd = NULL;
+        afr_private_t   *priv      = NULL;
+        int              found     = 0;
+
+        priv = this->private;
+
+        list_for_each_entry (locked_fd, &priv->saved_fds, list) {
+                if (locked_fd->fd == fd) {
+                        found = 1;
+                        break;
+                }
+        }
+
+        return found;
+}
+
+static int
+__afr_save_locked_fd (xlator_t *this, fd_t *fd)
+{
+        afr_private_t   *priv      = NULL;
+        afr_locked_fd_t *locked_fd = NULL;
+        int              ret       = 0;
+
+        priv = this->private;
+
+        locked_fd = GF_CALLOC (1, sizeof (*locked_fd),
+                               gf_afr_mt_locked_fd);
+        if (!locked_fd) {
+                ret = -1;
+                goto out;
+        }
+
+        locked_fd->fd = fd;
+        INIT_LIST_HEAD (&locked_fd->list);
+
+        list_add_tail (&locked_fd->list, &priv->saved_fds);
+
+out:
+        return ret;
+}
+
+int
+afr_save_locked_fd (xlator_t *this, fd_t *fd)
+{
+        afr_private_t   *priv      = NULL;
+        int              ret       = 0;
+
+        priv = this->private;
+
+        pthread_mutex_lock (&priv->mutex);
+        {
+                if (__is_fd_saved (this, fd)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "fd=%p already saved", fd);
+                        goto unlock;
+                }
+
+                ret = __afr_save_locked_fd (this, fd);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "fd=%p could not be saved", fd);
+                        goto unlock;
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&priv->mutex);
+
+        return ret;
+}
+
+static int
+afr_lock_recovery_cleanup (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local     = NULL;
+        afr_locked_fd_t *locked_fd = NULL;
+
+        local = frame->local;
+
+        locked_fd = local->locked_fd;
+
+        STACK_DESTROY (frame->root);
+        afr_local_cleanup (local, this);
+
+        afr_save_locked_fd (this, locked_fd->fd);
+
+        return 0;
+
+}
+
+static int
+afr_get_source_lock_recovery (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t  *fdctx        = NULL;
+        afr_private_t *priv         = NULL;
+        uint64_t      tmp           = 0;
+        int           i             = 0;
+        int           source_child  = -1;
+        int           ret           = 0;
+
+        priv = this->private;
+
+        ret = fd_ctx_get (fd, this, &tmp);
+        if (ret)
+                goto out;
+
+        fdctx = (afr_fd_ctx_t *) (long) tmp;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (fdctx->locked_on[i]) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Found lock recovery source=%d", i);
+                        source_child = i;
+                        break;
+                }
+        }
+
+out:
+        return source_child;
+
+}
+
+int32_t
+afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata);
+int32_t
+afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
+{
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int32_t source_child = 0;
+        struct gf_flock flock   = {0,};
+
+        local = frame->local;
+        priv  = this->private;
+
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "lock recovery failed");
+                goto cleanup;
+        }
+
+        source_child = local->source_child;
+
+        memcpy (&flock, lock, sizeof (*lock));
+
+        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
+                           (void *) (long) source_child,
+                           priv->children[source_child],
+                           priv->children[source_child]->fops->lk,
+                           local->fd, F_GETLK_FD, &flock, NULL);
+
+        return 0;
+
+cleanup:
+        afr_lock_recovery_cleanup (frame, this);
+        return 0;
+}
+
+int
+afr_recover_lock (call_frame_t *frame, xlator_t *this,
+                  struct gf_flock *flock)
+{
+        afr_local_t   *local             = NULL;
+        afr_private_t *priv              = NULL;
+        int32_t      lock_recovery_child = 0;
+
+        priv  = this->private;
+        local = frame->local;
+
+        lock_recovery_child = local->lock_recovery_child;
+
+        frame->root->lk_owner = flock->l_owner;
+
+        STACK_WIND_COOKIE (frame, afr_recover_lock_cbk,
+                           (void *) (long) lock_recovery_child,
+                           priv->children[lock_recovery_child],
+                           priv->children[lock_recovery_child]->fops->lk,
+                           local->fd, F_SETLK, flock, NULL);
+
+        return 0;
+}
+
+static int
+is_afr_lock_eol (struct gf_flock *lock)
+{
+        int ret = 0;
+
+        if ((lock->l_type == GF_LK_EOL))
+                ret = 1;
+
+        return ret;
+}
+
+int32_t
+afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
+{
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Failed to get locks on fd");
+                goto cleanup;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Got a lock on fd");
+
+        if (is_afr_lock_eol (lock)) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Reached EOL on locks on fd");
+                goto cleanup;
+        }
+
+        afr_recover_lock (frame, this, lock);
+
+        return 0;
+
+cleanup:
+        afr_lock_recovery_cleanup (frame, this);
+
+        return 0;
+}
+
+static int
+afr_lock_recovery (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local        = NULL;
+        afr_private_t *priv         = NULL;
+        fd_t          *fd           = NULL;
+        int            ret          = 0;
+        int32_t        source_child = 0;
+        struct gf_flock   flock        = {0,};
+
+        priv  = this->private;
+        local = frame->local;
+
+        fd = local->fd;
+
+        source_child = afr_get_source_lock_recovery (this, fd);
+        if (source_child < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not recover locks due to lock "
+                        "split brain");
+                ret = -1;
+                goto out;
+        }
+
+        local->source_child = source_child;
+
+        /* the flock can be zero filled as we're querying incrementally
+           the locks held on the fd.
+        */
+        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
+                           (void *) (long) source_child,
+                           priv->children[source_child],
+                           priv->children[source_child]->fops->lk,
+                           local->fd, F_GETLK_FD, &flock, NULL);
+
+out:
+        return ret;
+}
+
+
+static int
+afr_mark_fd_opened (xlator_t *this, fd_t *fd, int32_t child_index)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        uint64_t      tmp   = 0;
+        int           ret   = 0;
+
+        ret = fd_ctx_get (fd, this, &tmp);
+        if (ret)
+                goto out;
+
+        fdctx = (afr_fd_ctx_t *) (long) tmp;
+
+        fdctx->opened_on[child_index] = AFR_FD_OPENED;
+
+out:
+        return ret;
+}
+
+int32_t
+afr_lock_recovery_preopen_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                               int32_t op_ret, int32_t op_errno, fd_t *fd,
+                               dict_t *xdata)
+{
+        int32_t child_index = (long )cookie;
+        int ret = 0;
+
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Reopen during lock-recovery failed");
+                goto cleanup;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Open succeeded => proceed to recover locks");
+
+        ret = afr_lock_recovery (frame, this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Lock recovery failed");
+                goto cleanup;
+        }
+
+        ret = afr_mark_fd_opened (this, fd, child_index);
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Marking fd open failed");
+                goto cleanup;
+        }
+
+        return 0;
+
+cleanup:
+        afr_lock_recovery_cleanup (frame, this);
+        return 0;
+}
+
+static int
+afr_lock_recovery_preopen (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t *priv  = NULL;
+        afr_local_t   *local = NULL;
+        uint64_t       tmp   = 0;
+        afr_fd_ctx_t  *fdctx = NULL;
+        loc_t          loc   = {0,};
+        int32_t        child_index = 0;
+        int            ret   = 0;
+
+        priv  = this->private;
+        local = frame->local;
+
+        GF_ASSERT (local && local->fd);
+
+        ret = fd_ctx_get (local->fd, this, &tmp);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get the context of fd",
+                        uuid_utoa (local->fd->inode->gfid));
+        fdctx = (afr_fd_ctx_t *) (long) tmp;
+        /* TODO: instead we should return from the function */
+        GF_ASSERT (fdctx);
+
+        child_index = local->lock_recovery_child;
+
+        inode_path (local->fd->inode, NULL, (char **)&loc.path);
+        loc.name   = strrchr (loc.path, '/');
+        loc.inode  = inode_ref (local->fd->inode);
+        loc.parent = inode_parent (local->fd->inode, 0, NULL);
+
+
+        STACK_WIND_COOKIE (frame, afr_lock_recovery_preopen_cbk,
+                           (void *)(long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->open,
+                           &loc, fdctx->flags, local->fd, NULL);
+
+        return 0;
+}
+
+static int
+is_fd_opened (fd_t *fd, int32_t child_index)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        uint64_t      tmp = 0;
+        int           ret = 0;
+
+        ret = fd_ctx_get (fd, THIS, &tmp);
+        if (ret)
+                goto out;
+
+        fdctx = (afr_fd_ctx_t *) (long) tmp;
+
+        if (fdctx->opened_on[child_index] == AFR_FD_OPENED)
+                ret = 1;
+
+out:
+        return ret;
+}
+
+int
+afr_attempt_lock_recovery (xlator_t *this, int32_t child_index)
+{
+        call_frame_t    *frame      = NULL;
+        afr_private_t   *priv       = NULL;
+        afr_local_t     *local      = NULL;
+        afr_locked_fd_t *locked_fd  = NULL;
+        afr_locked_fd_t  *tmp       = NULL;
+        int              ret        = -1;
+        struct list_head locks_list = {0,};
+        int32_t          op_errno   = 0;
+
+
+        priv = this->private;
+
+        if (list_empty (&priv->saved_fds))
+                goto out;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        frame->local = local;
+
+        INIT_LIST_HEAD (&locks_list);
+
+        pthread_mutex_lock (&priv->mutex);
+        {
+                list_splice_init (&priv->saved_fds, &locks_list);
+        }
+        pthread_mutex_unlock (&priv->mutex);
+
+        list_for_each_entry_safe (locked_fd, tmp,
+                                  &locks_list, list) {
+
+                list_del_init (&locked_fd->list);
+
+                local->fd                  = fd_ref (locked_fd->fd);
+                local->lock_recovery_child = child_index;
+                local->locked_fd           = locked_fd;
+
+                if (!is_fd_opened (locked_fd->fd, child_index)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "attempting open before lock "
+                                "recovery");
+                        afr_lock_recovery_preopen (frame, this);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "attempting lock recovery "
+                                "without a preopen");
+                        afr_lock_recovery (frame, this);
+                }
+        }
+
+out:
+        if ((ret < 0) && frame)
+                AFR_STACK_DESTROY (frame);
+        return ret;
+}
+
+int
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
+                          unsigned int child_count)
+{
+        afr_local_t         *dst_local   = NULL;
+        afr_local_t         *src_local   = NULL;
+        afr_internal_lock_t *dst_lock    = NULL;
+        afr_internal_lock_t *src_lock    = NULL;
+        afr_inodelk_t       *dst_inodelk = NULL;
+        afr_inodelk_t       *src_inodelk = NULL;
+        int                 ret = -1;
+
+        src_local = src->local;
+        src_lock  = &src_local->internal_lock;
+        src_inodelk = afr_get_inodelk (src_lock, dom);
+        dst_local = dst->local;
+        dst_lock  = &dst_local->internal_lock;
+        dst_inodelk = afr_get_inodelk (dst_lock, dom);
+        if (!dst_inodelk || !src_inodelk)
+                goto out;
+        if (src_inodelk->locked_nodes) {
+                memcpy (dst_inodelk->locked_nodes, src_inodelk->locked_nodes,
+                        sizeof (*dst_inodelk->locked_nodes) * child_count);
+                memset (src_inodelk->locked_nodes, 0,
+                        sizeof (*src_inodelk->locked_nodes) * child_count);
+        }
+
+        dst_lock->transaction_lk_type = src_lock->transaction_lk_type;
+        dst_lock->selfheal_lk_type    = src_lock->selfheal_lk_type;
+        dst_inodelk->lock_count = src_inodelk->lock_count;
+        src_inodelk->lock_count = 0;
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
new file mode 100644
index 000000000..73594f265
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -0,0 +1,51 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __AFR_MEM_TYPES_H__
+#define __AFR_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_afr_mem_types_ {
+        gf_afr_mt_iovec  = gf_common_mt_end + 1,
+        gf_afr_mt_afr_fd_ctx_t,
+        gf_afr_mt_afr_private_t,
+        gf_afr_mt_int32_t,
+        gf_afr_mt_char,
+        gf_afr_mt_xattr_key,
+        gf_afr_mt_dict_t,
+        gf_afr_mt_xlator_t,
+        gf_afr_mt_iatt,
+        gf_afr_mt_int,
+        gf_afr_mt_afr_node_character,
+        gf_afr_mt_sh_diff_loop_state,
+        gf_afr_mt_uint8_t,
+        gf_afr_mt_loc_t,
+        gf_afr_mt_entry_name,
+        gf_afr_mt_pump_priv,
+        gf_afr_mt_locked_fd,
+        gf_afr_mt_inode_ctx_t,
+        gf_afr_fd_paused_call_t,
+        gf_afr_mt_crawl_data_t,
+        gf_afr_mt_brick_pos_t,
+        gf_afr_mt_shd_bool_t,
+        gf_afr_mt_shd_timer_t,
+        gf_afr_mt_shd_event_t,
+        gf_afr_mt_time_t,
+        gf_afr_mt_pos_data_t,
+        gf_afr_mt_reply_t,
+        gf_afr_mt_stats_t,
+        gf_afr_mt_shd_crawl_event_t,
+        gf_afr_mt_uint64_t,
+        gf_afr_mt_end
+};
+#endif
+
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
new file mode 100644
index 000000000..643a5d692
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -0,0 +1,382 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <libgen.h>
+#include <unistd.h>
+#include <fnmatch.h>
+#include <sys/time.h>
+#include <stdlib.h>
+#include <signal.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "afr.h"
+#include "dict.h"
+#include "xlator.h"
+#include "hashfn.h"
+#include "logging.h"
+#include "stack.h"
+#include "list.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "common-utils.h"
+#include "compat-errno.h"
+#include "compat.h"
+#include "byte-order.h"
+#include "statedump.h"
+
+#include "fd.h"
+
+#include "afr-inode-read.h"
+#include "afr-inode-write.h"
+#include "afr-dir-read.h"
+#include "afr-dir-write.h"
+#include "afr-transaction.h"
+#include "afr-self-heal.h"
+#include "afr-self-heal-common.h"
+
+int
+afr_stale_child_up (afr_local_t *local, xlator_t *this)
+{
+        int             i = 0;
+        afr_private_t   *priv = NULL;
+        int             up = -1;
+
+        priv = this->private;
+
+        if (!local->fresh_children)
+                local->fresh_children = afr_children_create (priv->child_count);
+        if (!local->fresh_children)
+                goto out;
+
+        afr_inode_get_read_ctx (this, local->fd->inode, local->fresh_children);
+        if (priv->child_count == afr_get_children_count (local->fresh_children,
+                                                         priv->child_count))
+                goto out;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->child_up[i])
+                        continue;
+                if (afr_is_child_present (local->fresh_children,
+                                          priv->child_count, i))
+                        continue;
+                up = i;
+                break;
+        }
+out:
+        return up;
+}
+
+void
+afr_perform_data_self_heal (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        inode_t         *inode = NULL;
+        int             st_child = -1;
+        char            reason[64] = {0};
+
+        local = frame->local;
+        sh = &local->self_heal;
+        inode = local->fd->inode;
+
+        if (!IA_ISREG (inode->ia_type))
+                goto out;
+
+        st_child = afr_stale_child_up (local, this);
+        if (st_child < 0)
+                goto out;
+
+        sh->do_data_self_heal          = _gf_true;
+        sh->do_metadata_self_heal      = _gf_true;
+        sh->do_gfid_self_heal          = _gf_true;
+        sh->do_missing_entry_self_heal = _gf_true;
+
+        snprintf (reason, sizeof (reason), "stale subvolume %d detected",
+                  st_child);
+        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
+                              reason, NULL, NULL);
+out:
+        return;
+}
+
+int
+afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t * local = frame->local;
+        afr_private_t *priv = NULL;
+
+        priv = this->private;
+        if (afr_open_only_data_self_heal (priv->data_self_heal))
+                afr_perform_data_self_heal (frame, this);
+        AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
+                          local->fd, xdata);
+        return 0;
+}
+
+
+int
+afr_open_cbk (call_frame_t *frame, void *cookie,
+              xlator_t *this, int32_t op_ret, int32_t op_errno,
+              fd_t *fd, dict_t *xdata)
+{
+        afr_local_t *  local       = NULL;
+        int            ret         = 0;
+        int            call_count  = -1;
+        int            child_index = (long) cookie;
+        afr_private_t *priv        = NULL;
+
+        priv = this->private;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                }
+
+                if (op_ret >= 0) {
+                        local->op_ret = op_ret;
+                        local->success_count++;
+
+                        ret = afr_child_fd_ctx_set (this, fd, child_index,
+                                                    local->cont.open.flags);
+                        if (ret) {
+                                local->op_ret = -1;
+                                local->op_errno = -ret;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                if ((local->cont.open.flags & O_TRUNC)
+                    && (local->op_ret >= 0)) {
+                        STACK_WIND (frame, afr_open_ftruncate_cbk,
+                                    this, this->fops->ftruncate,
+                                    fd, 0, NULL);
+                } else {
+                        if (afr_open_only_data_self_heal (priv->data_self_heal))
+                                afr_perform_data_self_heal (frame, this);
+                        AFR_STACK_UNWIND (open, frame, local->op_ret,
+                                          local->op_errno, local->fd, xdata);
+                }
+        }
+
+        return 0;
+}
+
+int
+afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          fd_t *fd, dict_t *xdata)
+{
+        afr_private_t * priv       = NULL;
+        afr_local_t *   local      = NULL;
+        int             i          = 0;
+        int             ret        = -1;
+        int32_t         call_count = 0;
+        int32_t         op_errno   = 0;
+        int32_t         wind_flags = flags & (~O_TRUNC);
+        //We can't let truncation to happen outside transaction.
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (this->private, out);
+        VALIDATE_OR_GOTO (loc, out);
+
+        priv = this->private;
+
+        if (flags & (O_CREAT|O_TRUNC)) {
+                QUORUM_CHECK(open,out);
+        }
+
+        if (afr_is_split_brain (this, loc->inode)) {
+                /* self-heal failed */
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to open as split brain seen, returning EIO");
+                op_errno = EIO;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        call_count   = local->call_count;
+        loc_copy (&local->loc, loc);
+
+        local->cont.open.flags   = flags;
+
+        local->fd = fd_ref (fd);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->open,
+                                           loc, wind_flags, fd, xdata);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, xdata);
+
+        return 0;
+}
+
+int
+afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, fd_t *fd,
+                         dict_t *xdata)
+{
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        afr_fd_ctx_t  *fd_ctx     = NULL;
+        int           call_count  = 0;
+        int           child_index = (long) cookie;
+
+        priv     = this->private;
+        local    = frame->local;
+
+        if (op_ret >= 0) {
+                gf_log (this->name, GF_LOG_DEBUG, "fd for %s opened "
+                        "successfully on subvolume %s", local->loc.path,
+                        priv->children[child_index]->name);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open %s "
+                        "on subvolume %s", local->loc.path,
+                        priv->children[child_index]->name);
+        }
+
+        fd_ctx = afr_fd_ctx_get (local->fd, this);
+        if (!fd_ctx) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to get fd context, %p", local->fd);
+                goto out;
+        }
+
+        LOCK (&local->fd->lock);
+        {
+                if (op_ret >= 0) {
+                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+                } else {
+                        fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+                }
+        }
+        UNLOCK (&local->fd->lock);
+out:
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                AFR_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+void
+afr_fix_open (xlator_t *this, fd_t *fd, size_t need_open_count, int *need_open)
+{
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
+        call_frame_t  *frame   = NULL;
+        afr_local_t   *local   = NULL;
+        int           ret      = -1;
+        int32_t       op_errno = 0;
+        afr_fd_ctx_t  *fd_ctx  = NULL;
+
+        priv  = this->private;
+
+        if (!afr_is_fd_fixable (fd) || !need_open || !need_open_count)
+                goto out;
+
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        local = frame->local;
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        local->loc.inode = inode_ref (fd->inode);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
+
+        local->fd = fd_ref (fd);
+        local->call_count = need_open_count;
+
+        gf_log (this->name, GF_LOG_DEBUG, "need open count: %zd",
+                need_open_count);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!need_open[i])
+                        continue;
+
+                if (IA_IFDIR == fd->inode->ia_type) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "opening fd for dir %s on subvolume %s",
+                                local->loc.path, priv->children[i]->name);
+
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
+                                           (void*) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->opendir,
+                                           &local->loc, local->fd,
+                                           NULL);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "opening fd for file %s on subvolume %s",
+                                local->loc.path, priv->children[i]->name);
+
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
+                                           (void *)(long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->open,
+                                           &local->loc,
+                                           fd_ctx->flags & (~O_TRUNC),
+                                           local->fd, NULL);
+                }
+
+        }
+        op_errno = 0;
+        ret = 0;
+out:
+        if (op_errno)
+                ret = -1; //For handling ALLOC_OR_GOTO
+        if (ret && frame)
+                AFR_STACK_DESTROY (frame);
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.c b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
new file mode 100644
index 000000000..83846f152
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
@@ -0,0 +1,837 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#include <openssl/md5.h>
+#include "glusterfs.h"
+#include "afr.h"
+#include "xlator.h"
+#include "dict.h"
+#include "xlator.h"
+#include "hashfn.h"
+#include "logging.h"
+#include "stack.h"
+#include "list.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "common-utils.h"
+#include "compat-errno.h"
+#include "compat.h"
+#include "byte-order.h"
+
+#include "afr-transaction.h"
+#include "afr-self-heal.h"
+#include "afr-self-heal-common.h"
+#include "afr-self-heal-algorithm.h"
+
+/*
+  This file contains the various self-heal algorithms
+*/
+
+static int
+sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
+                gf_boolean_t is_first_call, call_frame_t *old_loop_frame);
+static int
+sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame,
+                int32_t op_ret, int32_t op_errno);
+static int
+sh_destroy_frame (call_frame_t *frame, xlator_t *this)
+{
+        if (!frame)
+                goto out;
+
+        AFR_STACK_DESTROY (frame);
+out:
+        return 0;
+}
+
+static void
+sh_private_cleanup (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t             *local   = NULL;
+        afr_self_heal_t         *sh      = NULL;
+        afr_sh_algo_private_t   *sh_priv = NULL;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+
+        sh_priv = sh->private;
+        GF_FREE (sh_priv);
+}
+
+static int
+sh_number_of_writes_needed (unsigned char *write_needed, int child_count)
+{
+        int writes = 0;
+        int i      = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (write_needed[i])
+                        writes++;
+        }
+
+        return writes;
+}
+
+
+static int
+sh_loop_driver_done (call_frame_t *sh_frame, xlator_t *this,
+                     call_frame_t *last_loop_frame)
+{
+        afr_local_t             *local        = NULL;
+        afr_self_heal_t         *sh           = NULL;
+        afr_sh_algo_private_t   *sh_priv      = NULL;
+        int32_t                 total_blocks = 0;
+        int32_t                 diff_blocks  = 0;
+
+        local        = sh_frame->local;
+        sh           = &local->self_heal;
+        sh_priv      = sh->private;
+        if (sh_priv) {
+                total_blocks = sh_priv->total_blocks;
+                diff_blocks  = sh_priv->diff_blocks;
+        }
+
+        sh_private_cleanup (sh_frame, this);
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                GF_ASSERT (!last_loop_frame);
+                //loop_finish should have happened and the old_loop should be NULL
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "self-heal aborting on %s",
+                        local->loc.path);
+
+                local->self_heal.algo_abort_cbk (sh_frame, this);
+        } else {
+                GF_ASSERT (last_loop_frame);
+                if (diff_blocks == total_blocks) {
+                        gf_log (this->name, GF_LOG_DEBUG, "full self-heal "
+                                "completed on %s",local->loc.path);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "diff self-heal on %s: completed. "
+                                "(%d blocks of %d were different (%.2f%%))",
+                                local->loc.path, diff_blocks, total_blocks,
+                                ((diff_blocks * 1.0)/total_blocks) * 100);
+                }
+
+                sh->old_loop_frame = last_loop_frame;
+                local->self_heal.algo_completion_cbk (sh_frame, this);
+        }
+
+        return 0;
+}
+
+int
+sh_loop_finish (call_frame_t *loop_frame, xlator_t *this)
+{
+        afr_local_t             *loop_local = NULL;
+        afr_self_heal_t         *loop_sh = NULL;
+
+        if (!loop_frame)
+                goto out;
+
+        loop_local = loop_frame->local;
+        if (loop_local) {
+                loop_sh = &loop_local->self_heal;
+        }
+
+        if (loop_sh && loop_sh->data_lock_held) {
+                afr_sh_data_unlock (loop_frame, this, this->name,
+                                    sh_destroy_frame);
+        } else {
+                sh_destroy_frame (loop_frame, this);
+        }
+out:
+        return 0;
+}
+
+static int
+sh_loop_lock_success (call_frame_t *loop_frame, xlator_t *this)
+{
+        afr_local_t                 *loop_local = NULL;
+        afr_self_heal_t             *loop_sh    = NULL;
+
+        loop_local = loop_frame->local;
+        loop_sh = &loop_local->self_heal;
+
+        sh_loop_finish (loop_sh->old_loop_frame, this);
+        loop_sh->old_loop_frame = NULL;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Acquired lock for range %"PRIu64
+                " %"PRIu64, loop_sh->offset, loop_sh->block_size);
+        loop_sh->data_lock_held = _gf_true;
+        loop_sh->sh_data_algo_start (loop_frame, this);
+        return 0;
+}
+
+static int
+sh_loop_lock_failure (call_frame_t *loop_frame, xlator_t *this)
+{
+        call_frame_t                *sh_frame = NULL;
+        afr_local_t                 *loop_local = NULL;
+        afr_self_heal_t             *loop_sh    = NULL;
+
+        loop_local = loop_frame->local;
+        loop_sh = &loop_local->self_heal;
+        sh_frame = loop_sh->sh_frame;
+
+        gf_log (this->name, GF_LOG_ERROR, "failed lock for range %"PRIu64
+                " %"PRIu64, loop_sh->offset, loop_sh->block_size);
+        sh_loop_finish (loop_sh->old_loop_frame, this);
+        loop_sh->old_loop_frame = NULL;
+        sh_loop_return (sh_frame, this, loop_frame, -1, ENOTCONN);
+        return 0;
+}
+
+static int
+sh_loop_frame_create (call_frame_t *sh_frame, xlator_t *this,
+                      call_frame_t *old_loop_frame, call_frame_t **loop_frame)
+{
+        call_frame_t                *new_loop_frame = NULL;
+        afr_local_t                 *local          = NULL;
+        afr_self_heal_t             *sh             = NULL;
+        afr_local_t                 *new_loop_local = NULL;
+        afr_self_heal_t             *new_loop_sh    = NULL;
+        afr_private_t               *priv           = NULL;
+
+        GF_ASSERT (sh_frame);
+        GF_ASSERT (loop_frame);
+
+        *loop_frame = NULL;
+        local   = sh_frame->local;
+        sh      = &local->self_heal;
+        priv    = this->private;
+
+        new_loop_frame = copy_frame (sh_frame);
+        if (!new_loop_frame)
+                goto out;
+        //We want the frame to have same lk_owner as sh_frame
+        //so that locks translator allows conflicting locks
+        new_loop_local = afr_self_heal_local_init (local, this);
+        if (!new_loop_local)
+                goto out;
+        new_loop_frame->local = new_loop_local;
+
+        new_loop_sh = &new_loop_local->self_heal;
+        new_loop_sh->sources = memdup (sh->sources,
+                                       priv->child_count * sizeof (*sh->sources));
+        if (!new_loop_sh->sources)
+                goto out;
+        new_loop_sh->write_needed = GF_CALLOC (priv->child_count,
+                                               sizeof (*new_loop_sh->write_needed),
+                                               gf_afr_mt_char);
+        if (!new_loop_sh->write_needed)
+                goto out;
+        new_loop_sh->checksum = GF_CALLOC (priv->child_count, MD5_DIGEST_LENGTH,
+                                           gf_afr_mt_uint8_t);
+        if (!new_loop_sh->checksum)
+                goto out;
+        new_loop_sh->inode      = inode_ref (sh->inode);
+        new_loop_sh->sh_data_algo_start = sh->sh_data_algo_start;
+        new_loop_sh->source = sh->source;
+        new_loop_sh->active_sinks = sh->active_sinks;
+        new_loop_sh->healing_fd = fd_ref (sh->healing_fd);
+        new_loop_sh->file_has_holes = sh->file_has_holes;
+        new_loop_sh->old_loop_frame = old_loop_frame;
+        new_loop_sh->sh_frame = sh_frame;
+        *loop_frame = new_loop_frame;
+        return 0;
+out:
+        sh_destroy_frame (new_loop_frame, this);
+        return -ENOMEM;
+}
+
+static int
+sh_loop_start (call_frame_t *sh_frame, xlator_t *this, off_t offset,
+               call_frame_t *old_loop_frame)
+{
+        call_frame_t                *new_loop_frame = NULL;
+        afr_local_t                 *local          = NULL;
+        afr_self_heal_t             *sh             = NULL;
+        afr_local_t                 *new_loop_local = NULL;
+        afr_self_heal_t             *new_loop_sh    = NULL;
+        int                         ret             = 0;
+
+        GF_ASSERT (sh_frame);
+
+        local   = sh_frame->local;
+        sh      = &local->self_heal;
+
+        ret = sh_loop_frame_create (sh_frame, this, old_loop_frame,
+                                    &new_loop_frame);
+        if (ret)
+                goto out;
+        new_loop_local = new_loop_frame->local;
+        new_loop_sh = &new_loop_local->self_heal;
+        new_loop_sh->offset = offset;
+        new_loop_sh->block_size = sh->block_size;
+        afr_sh_data_lock (new_loop_frame, this, offset, new_loop_sh->block_size,
+                          _gf_true, this->name, sh_loop_lock_success, sh_loop_lock_failure);
+        return 0;
+out:
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        if (old_loop_frame)
+                sh_loop_finish (old_loop_frame, this);
+        sh_loop_return (sh_frame, this, new_loop_frame, -1, ENOMEM);
+        return 0;
+}
+
+static int
+sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
+                gf_boolean_t is_first_call, call_frame_t *old_loop_frame)
+{
+        afr_local_t *               local          = NULL;
+        afr_self_heal_t *           sh             = NULL;
+        afr_sh_algo_private_t       *sh_priv        = NULL;
+        gf_boolean_t                is_driver_done = _gf_false;
+        blksize_t                   block_size     = 0;
+        int                         loop           = 0;
+        off_t                       offset         = 0;
+        afr_private_t               *priv          = NULL;
+
+        priv    = this->private;
+        local   = sh_frame->local;
+        sh      = &local->self_heal;
+        sh_priv = sh->private;
+
+        LOCK (&sh_priv->lock);
+        {
+                if (!is_first_call)
+                        sh_priv->loops_running--;
+                offset = sh_priv->offset;
+                block_size = sh->block_size;
+                while ((!sh->eof_reached) &&
+                       (!is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) &&
+                     (sh_priv->loops_running < priv->data_self_heal_window_size)
+                       && (sh_priv->offset < sh->file_size)) {
+
+                        loop++;
+                        sh_priv->offset += block_size;
+                        sh_priv->loops_running++;
+
+                        if (!is_first_call)
+                                break;
+                }
+                if (0 == sh_priv->loops_running) {
+                        is_driver_done = _gf_true;
+                }
+        }
+        UNLOCK (&sh_priv->lock);
+
+        if (0 == loop) {
+                //loop finish does unlock, but the erasing of the pending
+                //xattrs needs to happen before that so do not finish the loop
+                if (is_driver_done &&
+                    !is_self_heal_failed (sh, AFR_CHECK_SPECIFIC))
+                        goto driver_done;
+                if (old_loop_frame) {
+                        sh_loop_finish (old_loop_frame, this);
+                        old_loop_frame = NULL;
+                }
+        }
+
+        //If we have more loops to form we should finish previous loop after
+        //the next loop lock
+        while (loop--) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                        // op failed in other loop, stop spawning more loops
+                        if (old_loop_frame) {
+                                sh_loop_finish (old_loop_frame, this);
+                                old_loop_frame = NULL;
+                        }
+                        sh_loop_driver (sh_frame, this, _gf_false, NULL);
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE, "spawning a loop "
+                                "for offset %"PRId64, offset);
+
+                        sh_loop_start (sh_frame, this, offset, old_loop_frame);
+                        old_loop_frame = NULL;
+                        offset += block_size;
+                }
+        }
+
+driver_done:
+        if (is_driver_done) {
+                sh_loop_driver_done (sh_frame, this, old_loop_frame);
+        }
+        return 0;
+}
+
+static int
+sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame,
+                int32_t op_ret, int32_t op_errno)
+{
+        afr_local_t *               loop_local = NULL;
+        afr_self_heal_t *           loop_sh    = NULL;
+        afr_local_t *               sh_local = NULL;
+        afr_self_heal_t            *sh       = NULL;
+
+        sh_local = sh_frame->local;
+        sh       = &sh_local->self_heal;
+
+        if (loop_frame) {
+                loop_local = loop_frame->local;
+                if (loop_local)
+                        loop_sh    = &loop_local->self_heal;
+                if (loop_sh)
+                        gf_log (this->name, GF_LOG_TRACE, "loop for offset "
+                                "%"PRId64" returned", loop_sh->offset);
+        }
+
+        if (op_ret == -1) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                afr_sh_set_error (sh, op_errno);
+                if (loop_frame) {
+                        sh_loop_finish (loop_frame, this);
+                        loop_frame = NULL;
+                }
+        }
+
+        sh_loop_driver (sh_frame, this, _gf_false, loop_frame);
+
+        return 0;
+}
+
+static int
+sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        afr_private_t *             priv        = NULL;
+        afr_local_t *               loop_local    = NULL;
+        afr_self_heal_t *           loop_sh       = NULL;
+        call_frame_t               *sh_frame    = NULL;
+        afr_local_t *               sh_local    = NULL;
+        afr_self_heal_t            *sh          = NULL;
+        int                         call_count  = 0;
+        int                         child_index = 0;
+
+        priv     = this->private;
+        loop_local = loop_frame->local;
+        loop_sh    = &loop_local->self_heal;
+
+        sh_frame = loop_sh->sh_frame;
+        sh_local = sh_frame->local;
+        sh       = &sh_local->self_heal;
+
+        child_index =  (long) cookie;
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "wrote %d bytes of data from %s to child %d, offset %"PRId64"",
+                op_ret, sh_local->loc.path, child_index, loop_sh->offset);
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "write to %s failed on subvolume %s (%s)",
+                        sh_local->loc.path,
+                        priv->children[child_index]->name,
+                        strerror (op_errno));
+
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                afr_sh_set_error (loop_sh, op_errno);
+        } else if (op_ret < loop_local->cont.writev.vector->iov_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "incomplete write to %s on subvolume %s "
+                        "(expected %lu, returned %d)", sh_local->loc.path,
+                        priv->children[child_index]->name,
+                        loop_local->cont.writev.vector->iov_len, op_ret);
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        }
+
+        call_count = afr_frame_return (loop_frame);
+
+        if (call_count == 0) {
+		iobref_unref(loop_local->cont.writev.iobref);
+
+                sh_loop_return (sh_frame, this, loop_frame,
+                                loop_sh->op_ret, loop_sh->op_errno);
+        }
+
+        return 0;
+}
+
+static void
+sh_prune_writes_needed (call_frame_t *sh_frame, call_frame_t *loop_frame,
+                        afr_private_t *priv)
+{
+        afr_local_t     *sh_local     = NULL;
+        afr_self_heal_t *sh           = NULL;
+        afr_local_t     *loop_local   = NULL;
+        afr_self_heal_t *loop_sh      = NULL;
+        int             i             = 0;
+
+        sh_local   = sh_frame->local;
+        sh         = &sh_local->self_heal;
+
+        if (!strcmp (sh->algo->name, "diff"))
+                return;
+
+        loop_local = loop_frame->local;
+        loop_sh    = &loop_local->self_heal;
+
+        /* full self-heal guarantees there exists atleast 1 file with size 0
+         * That means for other files we can preserve holes that come after
+         * its size before 'trim'
+         */
+        for (i = 0; i < priv->child_count; i++) {
+                if (loop_sh->write_needed[i] &&
+                    ((loop_sh->offset + 1) > sh->buf[i].ia_size))
+                        loop_sh->write_needed[i] = 0;
+        }
+}
+
+static int
+sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  struct iovec *vector, int32_t count, struct iatt *buf,
+                  struct iobref *iobref, dict_t *xdata)
+{
+        afr_private_t *               priv       = NULL;
+        afr_local_t *                 loop_local   = NULL;
+        afr_self_heal_t *             loop_sh      = NULL;
+        call_frame_t                 *sh_frame   = NULL;
+        int                           i          = 0;
+        int                           call_count = 0;
+        afr_local_t *                 sh_local   = NULL;
+        afr_self_heal_t *             sh      = NULL;
+
+        priv       = this->private;
+        loop_local = loop_frame->local;
+        loop_sh    = &loop_local->self_heal;
+
+        sh_frame = loop_sh->sh_frame;
+        sh_local = sh_frame->local;
+        sh       = &sh_local->self_heal;
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "read %d bytes of data from %s, offset %"PRId64"",
+                op_ret, loop_local->loc.path, loop_sh->offset);
+
+        if (op_ret <= 0) {
+                if (op_ret < 0) {
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                        gf_log (this->name, GF_LOG_ERROR, "read failed on %d "
+                                "for %s reason :%s", sh->source,
+                                sh_local->loc.path, strerror (errno));
+                } else {
+                        sh->eof_reached = _gf_true;
+                        gf_log (this->name, GF_LOG_DEBUG, "Eof reached for %s",
+                                sh_local->loc.path);
+                }
+                sh_loop_return (sh_frame, this, loop_frame, op_ret, op_errno);
+                goto out;
+        }
+
+        if (loop_sh->file_has_holes && iov_0filled (vector, count) == 0)
+                sh_prune_writes_needed (sh_frame, loop_frame, priv);
+
+        call_count = sh_number_of_writes_needed (loop_sh->write_needed,
+                                                 priv->child_count);
+        if (call_count == 0) {
+                sh_loop_return (sh_frame, this, loop_frame, 0, 0);
+                goto out;
+        }
+
+        loop_local->call_count = call_count;
+
+	/*
+	 * We only really need the request size at the moment, but the buffer
+	 * is required if we want to issue a retry in the event of a short write.
+	 * Therefore, we duplicate the vector and ref the iobref here...
+	 */
+	loop_local->cont.writev.vector = iov_dup(vector, count);
+	loop_local->cont.writev.iobref = iobref_ref(iobref);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!loop_sh->write_needed[i])
+                        continue;
+                STACK_WIND_COOKIE (loop_frame, sh_loop_write_cbk,
+                                   (void *) (long) i,
+                                   priv->children[i],
+                                   priv->children[i]->fops->writev,
+                                   loop_sh->healing_fd, vector, count,
+                                   loop_sh->offset, 0, iobref, NULL);
+
+                if (!--call_count)
+                        break;
+        }
+
+out:
+        return 0;
+}
+
+
+static int
+sh_loop_read (call_frame_t *loop_frame, xlator_t *this)
+{
+        afr_private_t           *priv       = NULL;
+        afr_local_t             *loop_local   = NULL;
+        afr_self_heal_t         *loop_sh      = NULL;
+
+        priv     = this->private;
+        loop_local = loop_frame->local;
+        loop_sh    = &loop_local->self_heal;
+
+        STACK_WIND_COOKIE (loop_frame, sh_loop_read_cbk,
+                           (void *) (long) loop_sh->source,
+                           priv->children[loop_sh->source],
+                           priv->children[loop_sh->source]->fops->readv,
+                           loop_sh->healing_fd, loop_sh->block_size,
+                           loop_sh->offset, 0, NULL);
+
+        return 0;
+}
+
+
+static int
+sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      uint32_t weak_checksum, uint8_t *strong_checksum,
+                      dict_t *xdata)
+{
+        afr_private_t                 *priv         = NULL;
+        afr_local_t                   *loop_local   = NULL;
+        afr_self_heal_t               *loop_sh      = NULL;
+        call_frame_t                  *sh_frame     = NULL;
+        afr_local_t                   *sh_local     = NULL;
+        afr_self_heal_t               *sh           = NULL;
+        afr_sh_algo_private_t         *sh_priv      = NULL;
+        int                           child_index  = 0;
+        int                           call_count   = 0;
+        int                           i            = 0;
+        int                           write_needed = 0;
+
+        priv  = this->private;
+
+        loop_local = loop_frame->local;
+        loop_sh    = &loop_local->self_heal;
+
+        sh_frame = loop_sh->sh_frame;
+        sh_local = sh_frame->local;
+        sh       = &sh_local->self_heal;
+
+        sh_priv = sh->private;
+
+        child_index = (long) cookie;
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "checksum on %s failed on subvolume %s (%s)",
+                        sh_local->loc.path, priv->children[child_index]->name,
+                        strerror (op_errno));
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        } else {
+                memcpy (loop_sh->checksum + child_index * MD5_DIGEST_LENGTH,
+                        strong_checksum, MD5_DIGEST_LENGTH);
+        }
+
+        call_count = afr_frame_return (loop_frame);
+
+        if (call_count == 0) {
+                for (i = 0; i < priv->child_count; i++) {
+                        if (sh->sources[i] || !sh_local->child_up[i])
+                                continue;
+
+                        if (memcmp (loop_sh->checksum + (i * MD5_DIGEST_LENGTH),
+                                    loop_sh->checksum + (sh->source * MD5_DIGEST_LENGTH),
+                                    MD5_DIGEST_LENGTH)) {
+                                /*
+                                  Checksums differ, so this block
+                                  must be written to this sink
+                                */
+
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "checksum on subvolume %s at offset %"
+                                        PRId64" differs from that on source",
+                                        priv->children[i]->name, loop_sh->offset);
+
+                                write_needed = loop_sh->write_needed[i] = 1;
+                        }
+                }
+
+                LOCK (&sh_priv->lock);
+                {
+                        sh_priv->total_blocks++;
+                        if (write_needed)
+                                sh_priv->diff_blocks++;
+                }
+                UNLOCK (&sh_priv->lock);
+
+                if (write_needed &&
+                    !is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                        sh_loop_read (loop_frame, this);
+                } else {
+                        sh_loop_return (sh_frame, this, loop_frame,
+                                        op_ret, op_errno);
+                }
+        }
+
+        return 0;
+}
+
+static int
+sh_diff_checksum (call_frame_t *loop_frame, xlator_t *this)
+{
+        afr_private_t           *priv         = NULL;
+        afr_local_t             *loop_local   = NULL;
+        afr_self_heal_t         *loop_sh      = NULL;
+        int                     call_count    = 0;
+        int                     i             = 0;
+
+        priv         = this->private;
+        loop_local   = loop_frame->local;
+        loop_sh      = &loop_local->self_heal;
+
+        call_count = loop_sh->active_sinks + 1;  /* sinks and source */
+
+        loop_local->call_count = call_count;
+
+        STACK_WIND_COOKIE (loop_frame, sh_diff_checksum_cbk,
+                           (void *) (long) loop_sh->source,
+                           priv->children[loop_sh->source],
+                           priv->children[loop_sh->source]->fops->rchecksum,
+                           loop_sh->healing_fd,
+                           loop_sh->offset, loop_sh->block_size, NULL);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (loop_sh->sources[i] || !loop_local->child_up[i])
+                        continue;
+
+                STACK_WIND_COOKIE (loop_frame, sh_diff_checksum_cbk,
+                                   (void *) (long) i,
+                                   priv->children[i],
+                                   priv->children[i]->fops->rchecksum,
+                                   loop_sh->healing_fd,
+                                   loop_sh->offset, loop_sh->block_size, NULL);
+
+                if (!--call_count)
+                        break;
+        }
+
+        return 0;
+}
+
+static int
+sh_full_read_write_to_sinks (call_frame_t *loop_frame, xlator_t *this)
+{
+        afr_private_t           *priv         = NULL;
+        afr_local_t             *loop_local   = NULL;
+        afr_self_heal_t         *loop_sh      = NULL;
+        int                     i             = 0;
+
+        priv         = this->private;
+        loop_local   = loop_frame->local;
+        loop_sh      = &loop_local->self_heal;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (loop_sh->sources[i] || !loop_local->child_up[i])
+                        continue;
+                loop_sh->write_needed[i] = 1;
+        }
+        sh_loop_read (loop_frame, this);
+        return 0;
+}
+
+afr_sh_algo_private_t*
+afr_sh_priv_init ()
+{
+        afr_sh_algo_private_t   *sh_priv = NULL;
+
+        sh_priv = GF_CALLOC (1, sizeof (*sh_priv),
+                             gf_afr_mt_afr_private_t);
+        if (!sh_priv)
+                goto out;
+
+        LOCK_INIT (&sh_priv->lock);
+out:
+        return sh_priv;
+}
+
+int
+afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src, char *dom,
+                      unsigned int child_count)
+{
+        afr_local_t             *dst_local   = NULL;
+        afr_self_heal_t         *dst_sh      = NULL;
+        afr_local_t             *src_local   = NULL;
+        afr_self_heal_t         *src_sh      = NULL;
+        int                     ret          = -1;
+
+        dst_local = dst->local;
+        dst_sh = &dst_local->self_heal;
+        src_local = src->local;
+        src_sh = &src_local->self_heal;
+        GF_ASSERT (src_sh->data_lock_held);
+        GF_ASSERT (!dst_sh->data_lock_held);
+        ret = afr_lk_transfer_datalock (dst, src, dom, child_count);
+        if (ret)
+                return ret;
+        src_sh->data_lock_held = _gf_false;
+        dst_sh->data_lock_held = _gf_true;
+        return 0;
+}
+
+int
+afr_sh_start_loops (call_frame_t *sh_frame, xlator_t *this,
+                    afr_sh_algo_fn sh_data_algo_start)
+{
+        call_frame_t            *first_loop_frame = NULL;
+        afr_local_t             *local   = NULL;
+        afr_self_heal_t         *sh      = NULL;
+        int                     ret      = 0;
+        afr_private_t           *priv    = NULL;
+
+        local = sh_frame->local;
+        sh    = &local->self_heal;
+        priv  = this->private;
+
+        sh->sh_data_algo_start = sh_data_algo_start;
+        local->call_count = 0;
+        ret = sh_loop_frame_create (sh_frame, this, NULL, &first_loop_frame);
+        if (ret)
+                goto out;
+        ret = afr_sh_transfer_lock (first_loop_frame, sh_frame, this->name,
+                                    priv->child_count);
+        if (ret)
+                goto out;
+        sh->private = afr_sh_priv_init ();
+        if (!sh->private) {
+                ret = -1;
+                goto out;
+        }
+        sh_loop_driver (sh_frame, this, _gf_true, first_loop_frame);
+        ret = 0;
+out:
+        if (ret) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                sh_loop_driver_done (sh_frame, this, NULL);
+        }
+        return 0;
+}
+
+int
+afr_sh_algo_diff (call_frame_t *sh_frame, xlator_t *this)
+{
+        afr_sh_start_loops (sh_frame, this, sh_diff_checksum);
+        return 0;
+}
+
+int
+afr_sh_algo_full (call_frame_t *sh_frame, xlator_t *this)
+{
+        afr_sh_start_loops (sh_frame, this, sh_full_read_write_to_sinks);
+        return 0;
+}
+
+struct afr_sh_algorithm afr_self_heal_algorithms[] = {
+        {.name = "full",  .fn = afr_sh_algo_full},
+        {.name = "diff",  .fn = afr_sh_algo_diff},
+        {0, 0},
+};
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.h b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
new file mode 100644
index 000000000..6b20789b1
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
@@ -0,0 +1,32 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __AFR_SELF_HEAL_ALGORITHM_H__
+#define __AFR_SELF_HEAL_ALGORITHM_H__
+
+typedef int (*afr_sh_algo_fn) (call_frame_t *frame,
+                               xlator_t *this);
+
+struct afr_sh_algorithm {
+        const char *name;
+        afr_sh_algo_fn fn;
+};
+
+extern struct afr_sh_algorithm afr_self_heal_algorithms[3];
+typedef struct {
+        gf_lock_t lock;
+        unsigned int loops_running;
+        off_t offset;
+
+        int32_t total_blocks;
+        int32_t diff_blocks;
+} afr_sh_algo_private_t;
+
+#endif /* __AFR_SELF_HEAL_ALGORITHM_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 45d065169..ef92b4205 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "glusterfs.h"
@@ -25,1049 +16,2797 @@
 #include "afr-transaction.h"
 #include "afr-self-heal-common.h"
 #include "afr-self-heal.h"
+#include "pump.h"
+
+#define ADD_FMT_STRING(msg, off, sh_str, status, print_log)                 \
+        do {                                                                \
+                if (AFR_SELF_HEAL_NOT_ATTEMPTED != status) {                \
+                        off += snprintf (msg + off, sizeof (msg) - off,     \
+                                         " "sh_str" self heal %s,",         \
+                                         get_sh_completion_status (status));\
+                        print_log = 1;                                      \
+                }                                                           \
+        } while (0)
+
+#define ADD_FMT_STRING_SYNC(msg, off, sh_str, status, print_log)            \
+        do {                                                                \
+                if (AFR_SELF_HEAL_SYNC_BEGIN == status ||                   \
+                    AFR_SELF_HEAL_FAILED == status)  {                      \
+                        off += snprintf (msg + off, sizeof (msg) - off,     \
+                                         " "sh_str" self heal %s,",         \
+                                         get_sh_completion_status (status));\
+                        print_log = 1;                                      \
+                }                                                           \
+        } while (0)
 
 
+void
+afr_sh_reset (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        memset (sh->child_errno, 0,
+                sizeof (*sh->child_errno) * priv->child_count);
+        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
+        memset (sh->parentbufs, 0,
+                sizeof (*sh->parentbufs) * priv->child_count);
+        memset (sh->success, 0, sizeof (*sh->success) * priv->child_count);
+        memset (sh->locked_nodes, 0,
+                sizeof (*sh->locked_nodes) * priv->child_count);
+        sh->active_sinks = 0;
+
+        afr_reset_xattr (sh->xattr, priv->child_count);
+}
+
+//Intersection[child]=1 if child is part of intersection
+void
+afr_children_intersection_get (int32_t *set1, int32_t *set2,
+                               int *intersection, unsigned int child_count)
+{
+        int                      i = 0;
+
+        memset (intersection, 0, sizeof (*intersection) * child_count);
+        for (i = 0; i < child_count; i++) {
+                intersection[i] = afr_is_child_present (set1, child_count, i)
+                                     && afr_is_child_present (set2, child_count,
+                                                              i);
+        }
+}
+
 /**
  * select_source - select a source and return it
- * TODO: take into account option 'favorite-child'
  */
 
 int
 afr_sh_select_source (int sources[], int child_count)
 {
-	int i;
-	for (i = 0; i < child_count; i++)
-		if (sources[i])
-			return i;
+        int i = 0;
+        for (i = 0; i < child_count; i++)
+                if (sources[i])
+                        return i;
 
-	return -1;
+        return -1;
 }
 
-
-/**
- * sink_count - return number of sinks in sources array
- */
-
-int
-afr_sh_sink_count (int sources[], int child_count)
+void
+afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this)
 {
-	int i;
-	int sinks = 0;
-	for (i = 0; i < child_count; i++)
-		if (!sources[i])
-			sinks++;
-	return sinks;
+        int              i = 0;
+        afr_local_t     *local      = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              active_sinks = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->sources[i] == 0 && local->child_up[i] == 1) {
+                        active_sinks++;
+                        sh->success[i] = 1;
+                } else if (sh->sources[i] == 1 && local->child_up[i] == 1) {
+                        sh->success[i] = 1;
+                }
+        }
+        sh->active_sinks = active_sinks;
 }
 
 int
 afr_sh_source_count (int sources[], int child_count)
 {
-	int i;
-	int nsource = 0;
+        int i = 0;
+        int nsource = 0;
 
-	for (i = 0; i < child_count; i++)
-		if (sources[i])
-			nsource++;
-	return nsource;
+        for (i = 0; i < child_count; i++)
+                if (sources[i])
+                        nsource++;
+        return nsource;
 }
 
-
-int
-afr_sh_supress_errenous_children (int sources[], int child_errno[],
-				  int child_count)
+void
+afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno)
 {
-	int i = 0;
+        sh->op_ret = -1;
+	sh->op_errno = afr_most_important_error(sh->op_errno, op_errno,
+						_gf_false);
+}
 
-	for (i = 0; i < child_count; i++) {
-		if (child_errno[i] && sources[i]) {
-			sources[i] = 0;
-		}
-	}
+void
+afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this)
+{
+        afr_private_t *  priv = this->private;
+        char            *buf  = NULL;
+        char            *ptr  = NULL;
+        int              i    = 0;
+        int              j    = 0;
 
-	return 0;
+        /* 10 digits per entry + 1 space + '[' and ']' */
+        buf = GF_MALLOC (priv->child_count * 11 + 8, gf_afr_mt_char);
+
+        for (i = 0; i < priv->child_count; i++) {
+                ptr = buf;
+                ptr += sprintf (ptr, "[ ");
+                for (j = 0; j < priv->child_count; j++) {
+                        ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
+                }
+                sprintf (ptr, "]");
+                gf_log (this->name, GF_LOG_DEBUG, "pending_matrix: %s", buf);
+        }
+
+        GF_FREE (buf);
 }
 
+char*
+afr_get_pending_matrix_str (int32_t *pending_matrix[], xlator_t *this)
+{
+        afr_private_t *  priv = this->private;
+        char            *buf  = NULL;
+        char            *ptr  = NULL;
+        int              i    = 0;
+        int              j    = 0;
+        int             child_count = priv->child_count;
+        char            *matrix_begin = "[ [ ";
+        char            *matrix_end = "] ]";
+        char            *seperator = "] [ ";
+        int             pending_entry_strlen = 12; //Including space after entry
+        int             matrix_begin_strlen = 0;
+        int             matrix_end_strlen = 0;
+        int             seperator_strlen = 0;
+        int             string_length = 0;
+        char            *msg = "- Pending matrix:  ";
+
+        /*
+         *  for a list of lists of [ [ a b ] [ c d ] ]
+         * */
+
+        matrix_begin_strlen = strlen (matrix_begin);
+        matrix_end_strlen = strlen (matrix_end);
+        seperator_strlen = strlen (seperator);
+        string_length = matrix_begin_strlen + matrix_end_strlen
+                        + (child_count -1) * seperator_strlen
+                        + (child_count * child_count * pending_entry_strlen);
+
+        buf = GF_CALLOC (1, 1 + strlen (msg) + string_length , gf_afr_mt_char);
+        if (!buf)
+                goto out;
+
+        ptr = buf;
+        ptr += sprintf (ptr, "%s", msg);
+        ptr += sprintf (ptr, "%s", matrix_begin);
+        for (i = 0; i < priv->child_count; i++) {
+                for (j = 0; j < priv->child_count; j++) {
+                        ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
+                }
+                if (i < priv->child_count -1)
+                        ptr += sprintf (ptr, "%s", seperator);
+        }
+
+        ptr += sprintf (ptr, "%s", matrix_end);
 
-int
-afr_sh_supress_empty_children (int sources[], dict_t *xattr[],
-			       struct stat *buf,
-			       int child_count, const char *key)
+out:
+        return buf;
+}
+
+void
+afr_sh_print_split_brain_log (int32_t *pending_matrix[], xlator_t *this,
+                              const char *loc)
 {
-	int      i = 0;
-	int32_t *pending = NULL;
-	int      ret = 0;
-	int      all_xattr_missing = 1;
+        char *buf      = NULL;
+        char *free_ptr = NULL;
 
-	/* if the file was created by afr with xattrs */
-	for (i = 0; i < child_count; i++) {
-		if (!xattr[i])
-			continue;
+        buf = afr_get_pending_matrix_str (pending_matrix, this);
+        if (buf)
+                free_ptr = buf;
+        else
+                buf = "";
 
-		ret = dict_get_ptr (xattr[i], (char *)key, VOID(&pending));
-		if (ret != 0) {
-			continue;
-		}
 
-		all_xattr_missing = 0;
-		break;
-	}
+        gf_log (this->name, GF_LOG_ERROR, "Unable to self-heal contents of '%s'"
+                " (possible split-brain). Please delete the file from all but "
+                "the preferred subvolume.%s", loc, buf);
+        GF_FREE (free_ptr);
+        return;
+}
 
-	if (all_xattr_missing) {
-		/* supress 0byte files.. this avoids empty file created
-		   by dir selfheal to overwrite the 'good' file */
-		for (i = 0; i < child_count; i++) {
-			if (!buf[i].st_size)
-				sources[i] = 0;
-		}
-		goto out;
-	}
 
+void
+afr_init_pending_matrix (int32_t **pending_matrix, size_t child_count)
+{
+        int             i   = 0;
+        int             j   = 0;
 
-	for (i = 0; i < child_count; i++) {
-		if (!xattr[i]) {
-			sources[i] = 0;
-			continue;
-		}
+        GF_ASSERT (pending_matrix);
 
-		ret = dict_get_ptr (xattr[i], (char *)key, VOID(&pending));
-		if (ret != 0) {
-			sources[i] = 0;
-			continue;
-		}
+        for (i = 0; i < child_count; i++) {
+                for (j = 0; j < child_count; j++) {
+                        pending_matrix[i][j] = 0;
+                }
+        }
+}
 
-		if (!pending) {
-			sources[i] = 0;
-			continue;
-		}
-	}
+void
+afr_mark_ignorant_subvols_as_pending (int32_t **pending_matrix,
+                                      unsigned char *ignorant_subvols,
+                                      size_t  child_count)
+{
+        int            i                = 0;
+        int            j                = 0;
+
+        GF_ASSERT (pending_matrix);
+        GF_ASSERT (ignorant_subvols);
+
+        for (i = 0; i < child_count; i++) {
+                if (ignorant_subvols[i]) {
+                        for (j = 0; j < child_count; j++) {
+                                if (!ignorant_subvols[j])
+                                        pending_matrix[j][i] += 1;
+                        }
+                }
+        }
+}
 
-out:
-	return 0;
+int
+afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
+                          unsigned char *ignorant_subvols,
+                          dict_t *xattr[], afr_transaction_type type,
+                          size_t child_count)
+{
+        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
+        int32_t        pending[3]       = {0,};
+        void          *pending_raw      = NULL;
+        int            ret              = -1;
+        int            i                = 0;
+        int            j                = 0;
+        int            k                = 0;
+
+        afr_init_pending_matrix (pending_matrix, child_count);
+
+        for (i = 0; i < child_count; i++) {
+                pending_raw = NULL;
+
+                for (j = 0; j < child_count; j++) {
+                        ret = dict_get_ptr (xattr[i], pending_key[j],
+                                            &pending_raw);
+
+                        if (ret != 0) {
+                                /*
+                                 * There is no xattr present. This means this
+                                 * subvolume should be considered an 'ignorant'
+                                 * subvolume.
+                                 */
+
+                                if (ignorant_subvols)
+                                        ignorant_subvols[i] = 1;
+                                continue;
+                        }
+
+                        memcpy (pending, pending_raw, sizeof(pending));
+                        k = afr_index_for_transaction_type (type);
+
+                        pending_matrix[i][j] = ntoh32 (pending[k]);
+                }
+        }
+
+        return ret;
 }
 
+typedef enum {
+        AFR_NODE_INVALID,
+        AFR_NODE_INNOCENT,
+        AFR_NODE_FOOL,
+        AFR_NODE_WISE,
+} afr_node_type;
 
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this)
+typedef struct {
+        afr_node_type type;
+        int           wisdom;
+} afr_node_character;
+
+
+static int
+afr_sh_is_innocent (int32_t *array, int child_count)
 {
-	afr_private_t * priv = this->private;
+        int i   = 0;
+        int ret = 1;   /* innocent until proven guilty */
 
-	char *buf = NULL;
-	char *ptr = NULL;
+        for (i = 0; i < child_count; i++) {
+                if (array[i]) {
+                        ret = 0;
+                        break;
+                }
+        }
 
-	int i, j;
+        return ret;
+}
 
-        /* 10 digits per entry + 1 space + '[' and ']' */
-	buf = MALLOC (priv->child_count * 11 + 8); 
 
-	for (i = 0; i < priv->child_count; i++) {
-		ptr = buf;
-		ptr += sprintf (ptr, "[ ");
-		for (j = 0; j < priv->child_count; j++) {
-			ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
-		}
-		ptr += sprintf (ptr, "]");
-		gf_log (this->name, GF_LOG_DEBUG,
-			"pending_matrix: %s", buf);
-	}
+static int
+afr_sh_is_fool (int32_t *array, int i, int child_count)
+{
+        return array[i];   /* fool if accuses itself */
+}
+
 
-	FREE (buf);
+static int
+afr_sh_is_wise (int32_t *array, int i, int child_count)
+{
+        return !array[i];  /* wise if does not accuse itself */
 }
 
 
-void
-afr_sh_build_pending_matrix (int32_t *pending_matrix[], dict_t *xattr[],
-			     int child_count, const char *key)
+static int
+afr_sh_all_nodes_innocent (afr_node_character *characters,
+                           int child_count)
 {
-	int i = 0;
-	int j = 0;
-	int32_t *pending = NULL;
-	int ret = -1;
+        int i   = 0;
+        int ret = 1;
 
-	/* start clean */
-	for (i = 0; i < child_count; i++) {
-		for (j = 0; j < child_count; j++) {
-			pending_matrix[i][j] = 0;
-		}
-	}
+        for (i = 0; i < child_count; i++) {
+                if (characters[i].type != AFR_NODE_INNOCENT) {
+                        ret = 0;
+                        break;
+                }
+        }
 
-	for (i = 0; i < child_count; i++) {
-		if (!xattr[i])
-			continue;
+        return ret;
+}
 
-		pending = NULL;
 
-		ret = dict_get_ptr (xattr[i], (char *) key,
-				    VOID(&pending));
-		if (ret != 0)
-			continue;
+static int
+afr_sh_wise_nodes_exist (afr_node_character *characters, int child_count)
+{
+        int i   = 0;
+        int ret = 0;
 
-		for (j = 0; j < child_count; j++) {
-			pending_matrix[i][j] = ntoh32 (pending[j]);
-		}
-	}
+        for (i = 0; i < child_count; i++) {
+                if (characters[i].type == AFR_NODE_WISE) {
+                        ret = 1;
+                        break;
+                }
+        }
+
+        return ret;
 }
 
 
-/**
- * mark_sources: Mark all 'source' nodes and return number of source
- * nodes found
+/*
+ * The 'wisdom' of a wise node is 0 if any other wise node accuses it.
+ * It is 1 if no other wise node accuses it.
+ * Only wise nodes with wisdom 1 are sources.
+ *
+ * If no nodes with wisdom 1 exist, a split-brain has occurred.
  */
 
-int
-afr_sh_mark_sources (int32_t *pending_matrix[], int sources[], int child_count)
+static void
+afr_sh_compute_wisdom (int32_t *pending_matrix[],
+                       afr_node_character characters[], int child_count)
 {
-	int i = 0;
-	int j = 0;
-
-	int nsources = 0;
-
+        int i = 0;
+        int j = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (characters[i].type == AFR_NODE_WISE) {
+                        characters[i].wisdom = 1;
+
+                        for (j = 0; j < child_count; j++) {
+                                if ((characters[j].type == AFR_NODE_WISE)
+                                    && pending_matrix[j][i]) {
+
+                                        characters[i].wisdom = 0;
+                                }
+                        }
+                }
+        }
+}
 
-	/* start clean */
-	for (i = 0; i < child_count; i++) {
-		sources[i] = 0;
-	}
 
-	/*
-	  Let's 'normalize' the pending matrix first,
-	  by disregarding all pending entries that refer
-	  to themselves
-	*/
-	for (i = 0; i < child_count; i++) {
-		pending_matrix[i][i] = 0;
-	}
+static int
+afr_sh_wise_nodes_conflict (afr_node_character *characters,
+                            int child_count)
+{
+        int i   = 0;
+        int ret = 1;
 
-	for (i = 0; i < child_count; i++) {
-		for (j = 0; j < child_count; j++) {
-			if (pending_matrix[j][i])
-				break;
-		}
+        for (i = 0; i < child_count; i++) {
+                if ((characters[i].type == AFR_NODE_WISE)
+                    && characters[i].wisdom == 1) {
 
-		if (j == child_count) {
-			nsources++;
-			sources[i] = 1;
-		}
-	}
+                        /* There is atleast one bona-fide wise node */
+                        ret = 0;
+                        break;
+                }
+        }
 
-	return nsources;
+        return ret;
 }
 
 
-void
-afr_sh_pending_to_delta (int32_t *pending_matrix[], int32_t *delta_matrix[],
-			 int success[], int child_count)
+static int
+afr_sh_mark_wisest_as_sources (int sources[],
+                               afr_node_character *characters,
+                               int child_count)
 {
-	int i = 0;
-	int j = 0;
+        int nsources = 0;
+        int i        = 0;
 
-	/* start clean */
-	for (i = 0; i < child_count; i++) {
-		for (j = 0; j < child_count; j++) {
-			delta_matrix[i][j] = 0;
-		}
-	}
+        for (i = 0; i < child_count; i++) {
+                if (characters[i].wisdom == 1) {
+                        sources[i] = 1;
+                        nsources++;
+                }
+        }
 
-	for (i = 0; i < child_count; i++) {
-		for (j = 0; j < child_count; j++) {
-			if (!success[j])
-				continue;
-			delta_matrix[i][j] = -pending_matrix[i][j];
-		}
-	}
+        return nsources;
 }
 
+static void
+afr_compute_witness_of_fools (int32_t *witnesses, int32_t **pending_matrix,
+                              afr_node_character *characters,
+                              int32_t child_count)
+{
+        int i       = 0;
+        int j       = 0;
+        int witness = 0;
+
+        GF_ASSERT (witnesses);
+        GF_ASSERT (pending_matrix);
+        GF_ASSERT (characters);
+        GF_ASSERT (child_count > 0);
+
+        for (i = 0; i < child_count; i++) {
+                if (characters[i].type != AFR_NODE_FOOL)
+                        continue;
+
+                witness = 0;
+                for (j = 0; j < child_count; j++) {
+                        if (i == j)
+                                continue;
+                        witness += pending_matrix[i][j];
+                }
+                witnesses[i] = witness;
+        }
+}
 
-int
-afr_sh_delta_to_xattr (int32_t *delta_matrix[], dict_t *xattr[],
-		       int child_count, const char *key)
+static int32_t
+afr_find_biggest_witness_among_fools (int32_t *witnesses,
+                                      afr_node_character *characters,
+                                      int32_t child_count)
 {
-	int i = 0;
-	int j = 0;
+        int i               = 0;
+        int biggest_witness = -1;
+        int biggest_witness_idx = -1;
+        int biggest_witness_cnt = -1;
+
+        GF_ASSERT (witnesses);
+        GF_ASSERT (characters);
+        GF_ASSERT (child_count > 0);
+
+        for (i = 0; i < child_count; i++) {
+                if (characters[i].type != AFR_NODE_FOOL)
+                        continue;
+
+                if (biggest_witness < witnesses[i]) {
+                        biggest_witness = witnesses[i];
+			biggest_witness_idx = i;
+			biggest_witness_cnt = 1;
+			continue;
+		}
 
-	int ret = 0;
+		if (biggest_witness == witnesses[i])
+			biggest_witness_cnt++;
+        }
 
-	int32_t *pending = 0;
+	if (biggest_witness_cnt != 1)
+		return -1;
 
-	for (i = 0; i < child_count; i++) {
-		if (!xattr[i])
-			continue;
+        return biggest_witness_idx;
+}
 
-		pending = CALLOC (sizeof (int32_t), child_count);
-		for (j = 0; j < child_count; j++) {
-			pending[j] = hton32 (delta_matrix[i][j]);
-		}
+int
+afr_mark_fool_as_source_by_witness (int32_t *sources, int32_t *witnesses,
+                                    afr_node_character *characters,
+                                    int32_t child_count, int32_t witness)
+{
+        int i        = 0;
+        int nsources = 0;
+
+        GF_ASSERT (sources);
+        GF_ASSERT (witnesses);
+        GF_ASSERT (characters);
+        GF_ASSERT (child_count > 0);
+
+        for (i = 0; i < child_count; i++) {
+                if (characters[i].type != AFR_NODE_FOOL)
+                        continue;
+
+                if (witness == witnesses[i]) {
+                        sources[i] = 1;
+                        nsources++;
+                }
+        }
+        return nsources;
+}
 
-		ret = dict_set_bin (xattr[i], (char *) key, pending,
-				    child_count * sizeof (int32_t));
-	}
 
+int
+afr_mark_fool_as_source_by_idx (int32_t *sources, int child_count, int idx)
+{
+	if (idx >= 0 && idx < child_count) {
+		sources[idx] = 1;
+		return 1;
+	}
 	return 0;
 }
 
 
-int
-afr_sh_has_metadata_pending (dict_t *xattr, int child_count, xlator_t *this)
+static int
+afr_find_largest_file_size (struct iatt *bufs, int32_t *success_children,
+			    int child_count)
 {
-	afr_private_t *priv = NULL;
-	int32_t       *pending = NULL;
-	void          *tmp_pending = NULL; /* This is required to remove 'type-punned' warnings from gcc */
-
-	int           ret = -1;
-	int            i  = 0;
+	int idx = -1;
+	int i = -1;
+	int child = -1;
+	uint64_t max_size = 0;
+        uint64_t min_size = 0;
+        int      num_children = 0;
 
-	priv = this->private;
+	for (i = 0; i < child_count; i++) {
+		if (success_children[i] == -1)
+			break;
 
-	ret = dict_get_ptr (xattr, AFR_METADATA_PENDING, &tmp_pending);
+		child = success_children[i];
+		if (bufs[child].ia_size > max_size) {
+			max_size = bufs[child].ia_size;
+			idx = child;
+		}
 
-	if (ret != 0)
-		return 0;
+                if ((num_children == 0) || (bufs[child].ia_size < min_size)) {
+                        min_size = bufs[child].ia_size;
+                }
 
-	pending = tmp_pending;
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == child_count)
-			continue;
-		if (pending[i])
-			return 1;
+                num_children++;
 	}
 
-	return 0;
+        /* If sizes are same for all of them, finding sources will have to
+         * happen with pending changelog. So return -1
+         */
+        if ((num_children > 1) && (min_size == max_size))
+                return -1;
+	return idx;
 }
 
 
-int
-afr_sh_has_data_pending (dict_t *xattr, int child_count, xlator_t *this)
+static int
+afr_find_newest_file (struct iatt *bufs, int32_t *success_children,
+		      int child_count)
 {
-	afr_private_t *priv = NULL;
-	int32_t       *pending = NULL;
-	void          *tmp_pending = NULL; /* This is required to remove 'type-punned' warnings from gcc */
+	int idx = -1;
+	int i = -1;
+	int child = -1;
+	uint64_t max_ctime = 0;
 
-	int          ret = -1;
-	int            i = 0;
+	for (i = 0; i < child_count; i++) {
+		if (success_children[i] == -1)
+			break;
 
-	priv = this->private;
+		child = success_children[i];
+		if (bufs[child].ia_ctime > max_ctime) {
+			max_ctime = bufs[child].ia_ctime;
+			idx = child;
+		}
+	}
 
-	ret = dict_get_ptr (xattr, AFR_DATA_PENDING, &tmp_pending);
+	return idx;
+}
 
-	if (ret != 0)
-		return 0;
 
-	pending = tmp_pending;
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == child_count)
-			continue;
-		if (pending[i])
-			return 1;
-	}
+static int
+afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
+                                     afr_node_character *characters,
+				     int32_t *success_children,
+                                     int child_count, struct iatt *bufs)
+{
+        int32_t       biggest_witness = 0;
+        int           nsources        = 0;
+        int32_t       *witnesses      = NULL;
+
+        GF_ASSERT (child_count > 0);
+
+	biggest_witness = afr_find_largest_file_size (bufs, success_children,
+						      child_count);
+	if (biggest_witness != -1)
+		goto found;
+
+        witnesses = GF_CALLOC (child_count, sizeof (*witnesses),
+                               gf_afr_mt_int32_t);
+        if (NULL == witnesses) {
+                nsources = -1;
+                goto out;
+        }
+
+        afr_compute_witness_of_fools (witnesses, pending_matrix, characters,
+                                      child_count);
+        biggest_witness = afr_find_biggest_witness_among_fools (witnesses,
+                                                                characters,
+                                                                child_count);
+	if (biggest_witness != -1)
+		goto found;
+
+	biggest_witness = afr_find_newest_file (bufs, success_children,
+						child_count);
+
+found:
+	nsources = afr_mark_fool_as_source_by_idx (sources, child_count,
+						   biggest_witness);
+out:
+        GF_FREE (witnesses);
+        return nsources;
+}
 
-	return 0;
+int
+afr_mark_child_as_source_by_uid (int32_t *sources, struct iatt *bufs,
+                                 int32_t *success_children,
+                                 unsigned int child_count, uint32_t uid)
+{
+        int     i        = 0;
+        int     nsources = 0;
+        int     child    = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (-1 == success_children[i])
+                        break;
+
+                child = success_children[i];
+                if (uid == bufs[child].ia_uid) {
+                        sources[child] = 1;
+                        nsources++;
+                }
+        }
+        return nsources;
 }
 
+int
+afr_get_child_with_lowest_uid (struct iatt *bufs, int32_t *success_children,
+                               unsigned int child_count)
+{
+        int     i        = 0;
+        int     smallest = -1;
+        int     child    = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (-1 == success_children[i])
+                        break;
+                child = success_children[i];
+                if ((smallest == -1) ||
+                    (bufs[child].ia_uid < bufs[smallest].ia_uid)) {
+                        smallest = child;
+                }
+        }
+        return smallest;
+}
+
+static int
+afr_sh_mark_lowest_uid_as_source (struct iatt *bufs, int32_t *success_children,
+                                  int child_count, int32_t *sources)
+{
+        int   nsources              = 0;
+        int   smallest              = 0;
+
+        smallest = afr_get_child_with_lowest_uid (bufs, success_children,
+                                                  child_count);
+        if (smallest < 0) {
+                nsources = -1;
+                goto out;
+        }
+        nsources = afr_mark_child_as_source_by_uid (sources, bufs,
+                                                    success_children, child_count,
+                                                    bufs[smallest].ia_uid);
+out:
+        return nsources;
+}
 
 int
-afr_sh_has_entry_pending (dict_t *xattr, int child_count, xlator_t *this)
+afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
+                                 struct iatt *bufs)
 {
-	afr_private_t *priv = NULL;
-	int32_t       *pending = NULL;
-	void          *tmp_pending = NULL; /* This is required to remove 'type-punned' warnings from gcc */
-	
-	int          ret = -1;
-	int            i = 0;
+        afr_private_t *priv = NULL;
+        int            i = 0;
+        int            child = -1;
+        int            read_child = -1;
+
+        priv = this->private;
+        for (i = 0; i < priv->child_count; i++) {
+                child = success_children[i];
+                if (child < 0)
+                        break;
+                if (read_child < 0)
+                        read_child = child;
+                else if (bufs[read_child].ia_size < bufs[child].ia_size)
+                        read_child = child;
+        }
+        return read_child;
+}
 
-	priv = this->private;
+int
+afr_sh_mark_zero_size_file_as_sink (struct iatt *bufs, int32_t *success_children,
+                                    int child_count, int32_t *sources)
+{
+        int             nsources = 0;
+        int             i = 0;
+        int             child = 0;
+        gf_boolean_t    sink_exists = _gf_false;
+        gf_boolean_t    source_exists = _gf_false;
+        int             source = -1;
+
+        for (i = 0; i < child_count; i++) {
+                child = success_children[i];
+                if (child < 0)
+                        break;
+                if (!bufs[child].ia_size) {
+                        sink_exists = _gf_true;
+                        continue;
+                }
+                if (!source_exists) {
+                        source_exists = _gf_true;
+                        source = child;
+                        continue;
+                }
+                if (bufs[source].ia_size != bufs[child].ia_size) {
+                        nsources = -1;
+                        goto out;
+                }
+        }
+        if (!source_exists && !sink_exists) {
+                nsources = -1;
+                goto out;
+        }
+
+        if (!source_exists || !sink_exists)
+                goto out;
+
+        for (i = 0; i < child_count; i++) {
+                child = success_children[i];
+                if (child < 0)
+                        break;
+                if (bufs[child].ia_size) {
+                        sources[child] = 1;
+                        nsources++;
+                }
+        }
+out:
+        return nsources;
+}
 
-	ret = dict_get_ptr (xattr, AFR_ENTRY_PENDING, &tmp_pending);
+char *
+afr_get_character_str (afr_node_type type)
+{
+        char *character = NULL;
+
+        switch (type) {
+        case AFR_NODE_INNOCENT:
+                character = "innocent";
+                break;
+        case AFR_NODE_FOOL:
+                character = "fool";
+                break;
+        case AFR_NODE_WISE:
+                character = "wise";
+                break;
+        default:
+                character = "invalid";
+                break;
+        }
+        return character;
+}
 
-	if (ret != 0)
-		return 0;
+afr_node_type
+afr_find_child_character_type (int32_t *pending_row, int32_t child,
+                               unsigned int child_count)
+{
+        afr_node_type type = AFR_NODE_INVALID;
 
-	pending = tmp_pending;
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == child_count)
-			continue;
-		if (pending[i])
-			return 1;
-	}
+        GF_ASSERT ((child >= 0) && (child < child_count));
 
-	return 0;
+        if (afr_sh_is_innocent (pending_row, child_count))
+                type = AFR_NODE_INNOCENT;
+        else if (afr_sh_is_fool (pending_row, child, child_count))
+                type = AFR_NODE_FOOL;
+        else if (afr_sh_is_wise (pending_row, child, child_count))
+                type = AFR_NODE_WISE;
+        return type;
 }
 
+int
+afr_build_sources (xlator_t *this, dict_t **xattr, struct iatt *bufs,
+                   int32_t **pending_matrix, int32_t *sources,
+                   int32_t *success_children, afr_transaction_type type,
+                   int32_t *subvol_status, gf_boolean_t ignore_ignorant)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heal_type      sh_type    = AFR_SELF_HEAL_INVALID;
+        int                     nsources   = -1;
+        unsigned char           *ignorant_subvols = NULL;
+        unsigned int            child_count = 0;
+
+        priv = this->private;
+        child_count = priv->child_count;
+
+        if (afr_get_children_count (success_children, priv->child_count) == 0)
+                goto out;
+
+        if (!ignore_ignorant) {
+                ignorant_subvols = GF_CALLOC (sizeof (*ignorant_subvols),
+                                              child_count, gf_afr_mt_char);
+                if (NULL == ignorant_subvols)
+                        goto out;
+        }
+
+        afr_build_pending_matrix (priv->pending_key, pending_matrix,
+                                  ignorant_subvols, xattr, type,
+                                  priv->child_count);
+
+        if (!ignore_ignorant)
+                afr_mark_ignorant_subvols_as_pending (pending_matrix,
+                                                      ignorant_subvols,
+                                                      priv->child_count);
+        sh_type = afr_self_heal_type_for_transaction (type);
+        if (AFR_SELF_HEAL_INVALID == sh_type)
+                goto out;
+
+        afr_sh_print_pending_matrix (pending_matrix, this);
+
+        nsources = afr_mark_sources (this, sources, pending_matrix, bufs,
+                                     sh_type, success_children, subvol_status);
+out:
+        GF_FREE (ignorant_subvols);
+        return nsources;
+}
 
+void
+afr_find_character_types (afr_node_character *characters,
+                          int32_t **pending_matrix, int32_t *success_children,
+                          unsigned int child_count)
+{
+        afr_node_type type  = AFR_NODE_INVALID;
+        int           child = 0;
+        int           i     = 0;
+
+        for (i = 0; i < child_count; i++) {
+                child = success_children[i];
+                if (child == -1)
+                        break;
+                type = afr_find_child_character_type (pending_matrix[child],
+                                                      child, child_count);
+                characters[child].type = type;
+        }
+}
 
+void
+afr_mark_success_children_sources (int32_t *sources, int32_t *success_children,
+                                   unsigned int child_count)
+{
+        int i = 0;
+        for (i = 0; i < child_count; i++) {
+                if (success_children[i] == -1)
+                        break;
+                sources[success_children[i]] = 1;
+        }
+}
 /**
- * is_matrix_zero - return true if pending matrix is all zeroes
+ * mark_sources: Mark all 'source' nodes and return number of source
+ * nodes found
+ *
+ * A node (a row in the pending matrix) belongs to one of
+ * three categories:
+ *
+ * M is the pending matrix.
+ *
+ * 'innocent' - M[i] is all zeroes
+ * 'fool'     - M[i] has i'th element = 1 (self-reference)
+ * 'wise'     - M[i] has i'th element = 0, others are 1 or 0.
+ *
+ * All 'innocent' nodes are sinks. If all nodes are innocent, no self-heal is
+ * needed.
+ *
+ * A 'wise' node can be a source. If two 'wise' nodes conflict, it is
+ * a split-brain. If one wise node refers to the other but the other doesn't
+ * refer back, the referrer is a source.
+ *
+ * All fools are sinks, unless there are no 'wise' nodes. In that case,
+ * one of the fools is made a source.
  */
 
 int
-afr_sh_is_matrix_zero (int32_t *pending_matrix[], int child_count)
+afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
+                  struct iatt *bufs, afr_self_heal_type type,
+                  int32_t *success_children, int32_t *subvol_status)
 {
-	int i, j;
+        /* stores the 'characters' (innocent, fool, wise) of the nodes */
+        afr_node_character *characters =  NULL;
+        int                nsources    = -1;
+        unsigned int       child_count = 0;
+        afr_private_t      *priv       = NULL;
+
+        priv = this->private;
+        child_count = priv->child_count;
+        characters = GF_CALLOC (sizeof (afr_node_character),
+                                child_count, gf_afr_mt_afr_node_character);
+        if (!characters)
+                goto out;
+
+        this = THIS;
+
+        /* start clean */
+        memset (sources, 0, sizeof (*sources) * child_count);
+        nsources = 0;
+        afr_find_character_types (characters, pending_matrix, success_children,
+                                  child_count);
+        if (afr_sh_all_nodes_innocent (characters, child_count)) {
+                switch (type) {
+                case AFR_SELF_HEAL_METADATA:
+                        nsources = afr_sh_mark_lowest_uid_as_source (bufs,
+                                                             success_children,
+                                                             child_count,
+                                                             sources);
+                        break;
+                case AFR_SELF_HEAL_DATA:
+                        nsources = afr_sh_mark_zero_size_file_as_sink (bufs,
+                                                             success_children,
+                                                             child_count,
+                                                             sources);
+                        if ((nsources < 0) && subvol_status)
+                                *subvol_status |= SPLIT_BRAIN;
+                        break;
+                default:
+                        break;
+                }
+                goto out;
+        }
+
+        if (afr_sh_wise_nodes_exist (characters, child_count)) {
+                afr_sh_compute_wisdom (pending_matrix, characters, child_count);
+
+                if (afr_sh_wise_nodes_conflict (characters, child_count)) {
+                        if (subvol_status)
+                                *subvol_status |= SPLIT_BRAIN;
+                        nsources = -1;
+                } else {
+                        nsources = afr_sh_mark_wisest_as_sources (sources,
+                                                                  characters,
+                                                                  child_count);
+                }
+        } else {
+                if (subvol_status)
+                        *subvol_status |= ALL_FOOLS;
+                nsources = afr_mark_biggest_of_fools_as_source (sources,
+                                                                pending_matrix,
+                                                                characters,
+								success_children,
+                                                                child_count, bufs);
+        }
 
-	for (i = 0; i < child_count; i++) 
-		for (j = 0; j < child_count; j++) 
-			if (pending_matrix[i][j]) 
-				return 0;
-	return 1;
-}
+out:
+        if (nsources == 0)
+                afr_mark_success_children_sources (sources, success_children,
+                                                   child_count);
+        GF_FREE (characters);
 
+        gf_log (this->name, GF_LOG_DEBUG, "Number of sources: %d", nsources);
+        return nsources;
+}
 
-int
-afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
+void
+afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
+                         int32_t *delta_matrix[], unsigned char success[],
+                         int child_count, afr_transaction_type type)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-//	memset (sh->child_errno, 0, sizeof (int) * priv->child_count);
-	memset (sh->buf, 0, sizeof (struct stat) * priv->child_count);
-	
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i])
-			dict_unref (sh->xattr[i]);
-		sh->xattr[i] = NULL;
-	}
+        int     tgt     = 0;
+        int     src     = 0;
+        int     value   = 0;
+
+        afr_build_pending_matrix (priv->pending_key, delta_matrix, NULL,
+                                  xattr, type, priv->child_count);
+
+        /*
+         * The algorithm here has two parts.  First, for each subvol indexed
+         * as tgt, we try to figure out what count everyone should have for it.
+         * If the self-heal succeeded, that's easy; the value is zero.
+         * Otherwise, the value is the maximum of the succeeding nodes' counts.
+         * Once we know the value, we loop through (possibly for a second time)
+         * setting each count to the difference so that when we're done all
+         * succeeding nodes will have the same count for tgt.
+         */
+        for (tgt = 0; tgt < priv->child_count; ++tgt) {
+                value = 0;
+                if (!success[tgt]) {
+                        /* Find the maximum. */
+                        for (src = 0; src < priv->child_count; ++src) {
+                                if (!success[src]) {
+                                        continue;
+                                }
+                                if (delta_matrix[src][tgt] > value) {
+                                        value = delta_matrix[src][tgt];
+                                }
+                        }
+                }
+                /* Force everyone who succeeded to the chosen value. */
+                for (src = 0; src < priv->child_count; ++src) {
+                        if (success[src]) {
+                                delta_matrix[src][tgt] = value
+                                                       - delta_matrix[src][tgt];
+                        }
+                        else {
+                                delta_matrix[src][tgt] = 0;
+                        }
+                }
+        }
+}
 
-	if (local->govinda_gOvinda) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"aborting selfheal of %s",
-			local->loc.path);
-		sh->completion_cbk (frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"proceeding to metadata check on %s",
-			local->loc.path);
-		afr_self_heal_metadata (frame, this);
-	}
 
-	return 0;
+int
+afr_sh_delta_to_xattr (xlator_t *this,
+                       int32_t *delta_matrix[], dict_t *xattr[],
+                       int child_count, afr_transaction_type type)
+{
+        int              i       = 0;
+        int              j       = 0;
+        int              k       = 0;
+        int              ret     = 0;
+        int32_t         *pending = NULL;
+        int32_t         *local_pending = NULL;
+        afr_private_t   *priv = NULL;
+
+        priv = this->private;
+        for (i = 0; i < child_count; i++) {
+                if (!xattr[i])
+                        continue;
+
+                local_pending = NULL;
+                for (j = 0; j < child_count; j++) {
+                        pending = GF_CALLOC (sizeof (int32_t), 3,
+                                             gf_afr_mt_int32_t);
+
+                        if (!pending) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to allocate pending entry "
+                                        "for %s[%d] on %s",
+                                        priv->pending_key[j], type,
+                                        priv->children[i]->name);
+                                continue;
+                        }
+                        /* 3 = data+metadata+entry */
+
+                        k = afr_index_for_transaction_type (type);
+
+                        pending[k] = hton32 (delta_matrix[i][j]);
+
+                        if (j == i) {
+                                local_pending = pending;
+                                continue;
+                        }
+                        ret = dict_set_bin (xattr[i], priv->pending_key[j],
+                                            pending,
+                                        AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Unable to set dict value.");
+                                GF_FREE (pending);
+                        }
+                }
+                if (local_pending) {
+                        ret = dict_set_bin (xattr[i], priv->pending_key[i],
+                                            local_pending,
+                                        AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Unable to set dict value.");
+                                GF_FREE (local_pending);
+                        }
+                }
+        }
+        return 0;
 }
 
 
 int
-sh_missing_entries_unlck_cbk (call_frame_t *frame, void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret, int32_t op_errno)
+afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        afr_sh_reset (frame, this);
+
+        if (local->unhealable) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "split brain found, aborting selfheal of %s",
+                        local->loc.path);
+        }
+
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                sh->completion_cbk (frame, this);
+        } else {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "proceeding to metadata check on %s",
+                        local->loc.path);
+                afr_self_heal_metadata (frame, this);
+        }
+
+        return 0;
+}
 
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+static int
+afr_sh_missing_entries_finish (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
 
-	LOCK (&frame->lock);
-	{
-	}
-	UNLOCK (&frame->lock);
+        local = frame->local;
+        int_lock = &local->internal_lock;
 
-	call_count = afr_frame_return (frame);
+        int_lock->lock_cbk = afr_sh_missing_entries_done;
+        afr_unlock (frame, this);
 
-	if (call_count == 0) {
-		afr_sh_missing_entries_done (frame, this);
-	}
+        return 0;
+}
 
-	return 0;
+int
+afr_sh_common_create (afr_self_heal_t *sh, unsigned int child_count)
+{
+        int     ret = -ENOMEM;
+        sh->buf = GF_CALLOC (child_count, sizeof (*sh->buf),
+                             gf_afr_mt_iatt);
+        if (!sh->buf)
+                goto out;
+        sh->parentbufs = GF_CALLOC (child_count, sizeof (*sh->parentbufs),
+                                    gf_afr_mt_iatt);
+        if (!sh->parentbufs)
+                goto out;
+        sh->child_errno = GF_CALLOC (child_count, sizeof (*sh->child_errno),
+                                     gf_afr_mt_int);
+        if (!sh->child_errno)
+                goto out;
+        sh->success_children = afr_children_create (child_count);
+        if (!sh->success_children)
+                goto out;
+        sh->fresh_children = afr_children_create (child_count);
+        if (!sh->fresh_children)
+                goto out;
+        sh->xattr = GF_CALLOC (child_count, sizeof (*sh->xattr),
+                               gf_afr_mt_dict_t);
+        if (!sh->xattr)
+                goto out;
+        ret = 0;
+out:
+        return ret;
 }
-			      
 
-static int
-sh_missing_entries_finish (call_frame_t *frame, xlator_t *this)
+void
+afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
+                                   xlator_t *this,
+                                   int32_t op_ret, int32_t op_errno,
+                                   inode_t *inode, struct iatt *buf,
+                                   dict_t *xattr, struct iatt *postparent,
+                                   loc_t *loc)
 {
-	afr_private_t      *priv = NULL;
-	afr_local_t        *local = NULL;
-	int                 i = 0;
-	int                 call_count = 0;
-	afr_self_heal_t    *sh = NULL;
+        int              child_index = 0;
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        priv = this->private;
+        sh   = &local->self_heal;
+        child_index = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == 0) {
+                        sh->buf[child_index] = *buf;
+                        sh->parentbufs[child_index] = *postparent;
+                        sh->success_children[sh->success_count] = child_index;
+                        sh->success_count++;
+                        sh->xattr[child_index] = dict_ref (xattr);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG, "path %s on subvolume"
+                                " %s => -1 (%s)", loc->path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        local->self_heal.child_errno[child_index] = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+        return;
+}
 
+gf_boolean_t
+afr_valid_ia_type (ia_type_t ia_type)
+{
+        switch (ia_type) {
+        case IA_IFSOCK:
+        case IA_IFREG:
+        case IA_IFBLK:
+        case IA_IFCHR:
+        case IA_IFIFO:
+        case IA_IFLNK:
+        case IA_IFDIR:
+                return _gf_true;
+        default:
+                return _gf_false;
+        }
+        return _gf_false;
+}
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+int
+afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
+                          int active_source, call_frame_t **impunge_frame)
+{
+        afr_local_t     *local         = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh    = NULL;
+        int32_t         op_errno       = 0;
+        afr_private_t   *priv          = NULL;
+        int             ret            = 0;
+        call_frame_t    *new_frame     = NULL;
+
+        op_errno = ENOMEM;
+        priv = this->private;
+        new_frame = copy_frame (frame);
+        if (!new_frame) {
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (impunge_local, out);
+
+        local = frame->local;
+        new_frame->local = impunge_local;
+        impunge_sh = &impunge_local->self_heal;
+        impunge_sh->sh_frame = frame;
+        impunge_sh->active_source = active_source;
+        impunge_local->child_up  = memdup (local->child_up,
+                                           sizeof (*local->child_up) *
+                                           priv->child_count);
+        if (!impunge_local->child_up)
+                goto out;
+
+        impunge_local->pending = afr_matrix_create (priv->child_count,
+                                                    AFR_NUM_CHANGE_LOGS);
+        if (!impunge_local->pending)
+                goto out;
+
+        ret = afr_sh_common_create (impunge_sh, priv->child_count);
+        if (ret) {
+                op_errno = -ret;
+                goto out;
+        }
+        op_errno = 0;
+        *impunge_frame = new_frame;
+out:
+        if (op_errno && new_frame)
+                AFR_STACK_DESTROY (new_frame);
+        return -op_errno;
+}
 
-	call_count = local->child_count;
+void
+afr_sh_missing_entry_call_impunge_recreate (call_frame_t *frame, xlator_t *this,
+                                            struct iatt *buf,
+                                            struct iatt *postparent,
+                                            afr_impunge_done_cbk_t impunge_done)
+{
+        call_frame_t    *impunge_frame = NULL;
+        afr_local_t     *local = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_self_heal_t *impunge_sh = NULL;
+        int             ret = 0;
+        unsigned int    enoent_count = 0;
+        afr_private_t   *priv = NULL;
+        int             i = 0;
+        int32_t         op_errno = 0;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+        priv  = this->private;
+
+        enoent_count = afr_errno_count (NULL, sh->child_errno,
+                                        priv->child_count, ENOENT);
+        if (!enoent_count) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no missing files - %s. proceeding to metadata check",
+                        local->loc.path);
+                goto out;
+        }
+        sh->impunge_done = impunge_done;
+        ret = afr_impunge_frame_create (frame, this, sh->source, &impunge_frame);
+        if (ret)
+                goto out;
+        impunge_local = impunge_frame->local;
+        impunge_sh    = &impunge_local->self_heal;
+        loc_copy (&impunge_local->loc, &local->loc);
+        ret = afr_build_parent_loc (&impunge_sh->parent_loc,
+                                    &impunge_local->loc, &op_errno);
+        if (ret) {
+                ret = -op_errno;
+                goto out;
+        }
+        impunge_local->call_count = enoent_count;
+        impunge_sh->entrybuf = sh->buf[sh->source];
+        impunge_sh->parentbuf = sh->parentbufs[sh->source];
+        for (i = 0; i < priv->child_count; i++) {
+                if (!impunge_local->child_up[i]) {
+                        impunge_sh->child_errno[i] = ENOTCONN;
+                        continue;
+                }
+                if (sh->child_errno[i] != ENOENT) {
+                        impunge_sh->child_errno[i] = EEXIST;
+                        continue;
+                }
+        }
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->child_errno[i] != ENOENT)
+                        continue;
+                afr_sh_entry_impunge_create (impunge_frame, this, i);
+                enoent_count--;
+        }
+        GF_ASSERT (!enoent_count);
+        return;
+out:
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "impunge of %s failed, "
+                        "reason: %s", local->loc.path, strerror (-ret));
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        }
+        afr_sh_missing_entries_finish (frame, this);
+}
 
-	local->call_count = call_count;
+int
+afr_sh_create_entry_cbk (call_frame_t *frame, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        if (op_ret < 0)
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        afr_sh_missing_entries_finish (frame, this);
+        return 0;
+}
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"unlocking %"PRId64"/%s on subvolume %s",
-				sh->parent_loc.inode->ino, local->loc.name,
-				priv->children[i]->name);
+static int
+sh_missing_entries_create (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        int              type = 0;
+        struct iatt     *buf = NULL;
+        struct iatt     *postparent = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        buf = &sh->buf[sh->source];
+        postparent = &sh->parentbufs[sh->source];
+
+        type = buf->ia_type;
+        if (!afr_valid_ia_type (type)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: unknown file type: 0%o", local->loc.path, type);
+                afr_set_local_for_unhealable (local);
+                afr_sh_missing_entries_finish (frame, this);
+                goto out;
+        }
+
+        afr_sh_missing_entry_call_impunge_recreate (frame, this,
+                                                    buf, postparent,
+                                                    afr_sh_create_entry_cbk);
+out:
+        return 0;
+}
 
-			STACK_WIND (frame, sh_missing_entries_unlck_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->entrylk,
-				    &sh->parent_loc, local->loc.name,
-				    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+void
+afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
+                                    int32_t op_ret, int32_t op_errno)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        ia_type_t       ia_type = IA_INVAL;
+        int32_t         nsources = 0;
+        loc_t           *loc = NULL;
+        int32_t         subvol_status = 0;
+        afr_transaction_type txn_type = AFR_DATA_TRANSACTION;
+        gf_boolean_t    split_brain = _gf_false;
+        int             read_child = -1;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+        loc = &local->loc;
+
+        if (op_ret < 0) {
+                if (op_errno == EIO) {
+                        afr_set_local_for_unhealable (local);
+                }
+                // EIO can happen if finding the fresh parent dir failed
+                goto out;
+        }
+
+        //now No chance for the ia_type to conflict
+        ia_type = sh->buf[sh->success_children[0]].ia_type;
+        txn_type = afr_transaction_type_get (ia_type);
+        nsources = afr_build_sources (this, sh->xattr, sh->buf,
+                                      sh->pending_matrix, sh->sources,
+                                      sh->success_children, txn_type,
+                                      &subvol_status, _gf_false);
+        if (nsources < 0) {
+                gf_log (this->name, GF_LOG_INFO, "No sources for dir of %s,"
+                        " in missing entry self-heal, continuing with the rest"
+                        " of the self-heals", local->loc.path);
+                if (subvol_status & SPLIT_BRAIN) {
+                        split_brain = _gf_true;
+                        switch (txn_type) {
+                        case AFR_DATA_TRANSACTION:
+                                nsources = 1;
+                                sh->sources[sh->success_children[0]] = 1;
+                                break;
+                        case AFR_ENTRY_TRANSACTION:
+                                read_child = afr_get_no_xattr_dir_read_child
+                                                          (this,
+                                                           sh->success_children,
+                                                           sh->buf);
+                                sh->sources[read_child] = 1;
+                                nsources = 1;
+                                break;
+                        default:
+                                op_errno = EIO;
+                                goto out;
+                        }
+                } else {
+                        op_errno = EIO;
+                        goto out;
+                }
+        }
+
+        afr_get_fresh_children (sh->success_children, sh->sources,
+                                sh->fresh_children, priv->child_count);
+        sh->source = sh->fresh_children[0];
+        if (sh->source == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
+                op_errno = EIO;
+                goto out;
+        }
+
+        if (sh->gfid_sh_success_cbk)
+                sh->gfid_sh_success_cbk (frame, this);
+        sh->type = sh->buf[sh->source].ia_type;
+        if (uuid_is_null (loc->inode->gfid))
+                uuid_copy (loc->gfid, sh->buf[sh->source].ia_gfid);
+        if (split_brain) {
+                afr_sh_missing_entries_finish (frame, this);
+        } else {
+                sh_missing_entries_create (frame, this);
+        }
+        return;
+out:
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        afr_sh_set_error (sh, op_errno);
+        afr_sh_missing_entries_finish (frame, this);
+        return;
+}
 
-			if (!--call_count)
-				break;
-		}
-	}
-	return 0;
+static int
+afr_sh_common_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xattr,
+                          struct iatt *postparent)
+{
+        int                     call_count = 0;
+        afr_local_t             *local = NULL;
+        afr_self_heal_t         *sh    = NULL;
+        afr_private_t           *priv  = NULL;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+        priv  = this->private;
+
+        afr_sh_common_lookup_resp_handler (frame, cookie, this, op_ret,
+                                           op_errno, inode, buf, xattr,
+                                           postparent, &sh->lookup_loc);
+        call_count = afr_frame_return (frame);
+
+        if (call_count)
+                goto out;
+        op_ret = -1;
+        if (!sh->success_count) {
+                op_errno = afr_resultant_errno_get (NULL, sh->child_errno,
+                                                    priv->child_count);
+                gf_log (this->name, GF_LOG_ERROR, "Failed to lookup %s, "
+                        "reason %s", sh->lookup_loc.path,
+                        strerror (op_errno));
+                goto done;
+        }
+
+        if ((sh->lookup_flags & AFR_LOOKUP_FAIL_CONFLICTS) &&
+            (afr_conflicting_iattrs (sh->buf, sh->success_children,
+                                     priv->child_count,
+                                     sh->lookup_loc.path, this->name))) {
+                op_errno = EIO;
+                gf_log (this->name, GF_LOG_ERROR, "Conflicting entries "
+                        "for %s", sh->lookup_loc.path);
+                goto done;
+        }
+
+        if ((sh->lookup_flags & AFR_LOOKUP_FAIL_MISSING_GFIDS) &&
+            (afr_gfid_missing_count (this->name, sh->success_children,
+                                     sh->buf, priv->child_count,
+                                     sh->lookup_loc.path))) {
+                op_errno = ENODATA;
+                gf_log (this->name, GF_LOG_ERROR, "Missing Gfids "
+                        "for %s", sh->lookup_loc.path);
+                goto done;
+        }
+        op_ret = 0;
+
+done:
+        sh->lookup_done (frame, this, op_ret, op_errno);
+out:
+        return 0;
 }
 
+int
+afr_sh_remove_entry_cbk (call_frame_t *frame, xlator_t *this, int child,
+                         int32_t op_ret, int32_t op_errno)
+{
+        int             call_count = 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        GF_ASSERT (sh->post_remove_call);
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "purge entry %s failed, on child %d reason, %s",
+                        local->loc.path, child, strerror (op_errno));
+                LOCK (&frame->lock);
+                {
+                        afr_sh_set_error (sh, EIO);
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                }
+                UNLOCK (&frame->lock);
+        }
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                sh->post_remove_call (frame, this);
+        return 0;
+}
 
-static int
-sh_destroy_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int op_errno, struct stat *stbuf)
+void
+afr_sh_call_entry_expunge_remove (call_frame_t *frame, xlator_t *this,
+                                  int child_index, struct iatt *buf,
+                                  struct iatt *parentbuf,
+                                  afr_expunge_done_cbk_t expunge_done)
 {
-	STACK_DESTROY (frame->root);
-	return 0;
+        call_frame_t    *expunge_frame = NULL;
+        afr_local_t     *local = NULL;
+        afr_local_t     *expunge_local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_self_heal_t *expunge_sh = NULL;
+        int32_t         op_errno = 0;
+        int             ret = 0;
+
+        expunge_frame = copy_frame (frame);
+        if (!expunge_frame) {
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
+
+        local = frame->local;
+        sh = &local->self_heal;
+        expunge_frame->local = expunge_local;
+        expunge_sh = &expunge_local->self_heal;
+        expunge_sh->sh_frame = frame;
+        loc_copy (&expunge_local->loc, &local->loc);
+        ret = afr_build_parent_loc (&expunge_sh->parent_loc,
+                                    &expunge_local->loc, &op_errno);
+        if (ret) {
+                ret = -op_errno;
+                goto out;
+        }
+        sh->expunge_done = expunge_done;
+        afr_sh_entry_expunge_remove (expunge_frame, this, child_index, buf,
+                                     parentbuf);
+        return;
+out:
+        gf_log (this->name, GF_LOG_ERROR, "Expunge of %s failed, reason: %s",
+                local->loc.path, strerror (op_errno));
+        expunge_done (frame, this, child_index, -1, op_errno);
 }
 
+void
+afr_sh_remove_stale_lookup_info (afr_self_heal_t *sh, int32_t *success_children,
+                                 int32_t *fresh_children,
+                                 unsigned int child_count)
+{
+        int     i = 0;
+
+        for (i = 0; i < child_count; i++) {
+                if (afr_is_child_present (success_children, child_count, i) &&
+                    !afr_is_child_present (fresh_children, child_count, i)) {
+                        sh->child_errno[i] = ENOENT;
+                        GF_ASSERT (sh->xattr[i]);
+                        dict_unref (sh->xattr[i]);
+                        sh->xattr[i] = NULL;
+                }
+        }
+}
 
-static int
-sh_missing_entries_newentry_cbk (call_frame_t *frame, void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret, int32_t op_errno,
-				 inode_t *inode, struct stat *stbuf)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	call_frame_t    *chown_frame = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-	struct stat     *buf = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	buf = &sh->buf[sh->source];
-	child_index = (long) cookie;
-
-	if (op_ret == 0) {
-		chown_frame = copy_frame (frame);
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"chown %s to %d %d on subvolume %s",
-			local->loc.path, buf->st_uid, buf->st_gid,
-			priv->children[child_index]->name);
-
-		STACK_WIND (chown_frame, sh_destroy_cbk,
-			    priv->children[child_index],
-			    priv->children[child_index]->fops->chown,
-			    &local->loc,
-			    buf->st_uid, buf->st_gid);
-	}
+int
+afr_sh_purge_stale_entries_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
+        afr_private_t       *priv     = NULL;
+
+        local    = frame->local;
+        sh       = &local->self_heal;
+        priv     = this->private;
+
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                afr_sh_missing_entries_finish (frame, this);
+        } else {
+                if (afr_gfid_missing_count (this->name, sh->fresh_children,
+                                            sh->buf, priv->child_count,
+                                            local->loc.path)) {
+                        afr_sh_common_lookup (frame, this, &local->loc,
+                                              afr_sh_missing_entries_lookup_done,
+                                              sh->sh_gfid_req,
+                                              AFR_LOOKUP_FAIL_CONFLICTS|
+                                              AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                              NULL);
+                } else {
+                        //No need to set gfid so goto missing entries lookup done
+                        //Behave as if you have done the lookup
+                        afr_sh_remove_stale_lookup_info (sh,
+                                                         sh->success_children,
+                                                         sh->fresh_children,
+                                                         priv->child_count);
+                        afr_children_copy (sh->success_children,
+                                           sh->fresh_children,
+                                           priv->child_count);
+                        afr_sh_missing_entries_lookup_done (frame, this, 0, 0);
+                }
+        }
+        return 0;
+}
 
-	LOCK (&frame->lock);
-	{
-	}
-	UNLOCK (&frame->lock);
+gf_boolean_t
+afr_sh_purge_entry_condition (afr_local_t *local, afr_private_t *priv,
+                              int child)
+{
+        afr_self_heal_t *sh = NULL;
 
-	call_count = afr_frame_return (frame);
+        sh = &local->self_heal;
 
-	if (call_count == 0) {
-		sh_missing_entries_finish (frame, this);
-	}
+        if (local->child_up[child] &&
+            (!afr_is_child_present (sh->fresh_parent_dirs, priv->child_count,
+                                    child))
+            && (sh->child_errno[child] != ENOENT))
+                return _gf_true;
 
-	return 0;
+        return _gf_false;
 }
 
+gf_boolean_t
+afr_sh_purge_stale_entry_condition (afr_local_t *local, afr_private_t *priv,
+                                    int child)
+{
+        afr_self_heal_t *sh = NULL;
 
-static int
-sh_missing_entries_mknod (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-	int              enoent_count = 0;
-	int              call_count = 0;
-	mode_t           st_mode = 0;
-	dev_t            st_dev = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sh->child_errno[i] == ENOENT)
-			enoent_count++;
-
-	call_count = enoent_count;
-	local->call_count = call_count;
-
-	st_mode = sh->buf[sh->source].st_mode;
-	st_dev  = sh->buf[sh->source].st_dev;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"mknod %s mode 0%o on %d subvolumes",
-		local->loc.path, st_mode, enoent_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i] == ENOENT) {
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_newentry_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->mknod,
-					   &local->loc, st_mode, st_dev);
-			if (!--call_count)
-				break;
-		}
-	}
+        sh = &local->self_heal;
 
-	return 0;
+        if (local->child_up[child] &&
+            (!afr_is_child_present (sh->fresh_children, priv->child_count,
+                                    child))
+             && (sh->child_errno[child] != ENOENT))
+                return _gf_true;
+
+        return _gf_false;
 }
 
+void
+afr_sh_purge_entry_common (call_frame_t *frame, xlator_t *this,
+                           gf_boolean_t purge_condition (afr_local_t *local,
+                                                         afr_private_t *priv,
+                                                         int child))
+{
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
+        afr_self_heal_t *sh = NULL;
+        int             i = 0;
+        int             call_count = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (purge_condition (local, priv, i))
+                        call_count++;
+        }
+
+        if (call_count == 0) {
+                sh->post_remove_call (frame, this);
+                goto out;
+        }
+
+        local->call_count = call_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!purge_condition (local, priv, i))
+                        continue;
+                gf_log (this->name, GF_LOG_INFO, "purging the stale entry %s "
+                        "on %s", local->loc.path, priv->children[i]->name);
+                afr_sh_call_entry_expunge_remove (frame, this,
+                                                  (long) i, &sh->buf[i],
+                                                  &sh->parentbufs[i],
+                                                  afr_sh_remove_entry_cbk);
+        }
+out:
+        return;
+}
 
-static int
-sh_missing_entries_mkdir (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-	int              enoent_count = 0;
-	int              call_count = 0;
-	mode_t           st_mode = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sh->child_errno[i] == ENOENT)
-			enoent_count++;
-
-	call_count = enoent_count;
-	local->call_count = call_count;
-
-	st_mode = sh->buf[sh->source].st_mode;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"mkdir %s mode 0%o on %d subvolumes",
-		local->loc.path, st_mode, enoent_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i] == ENOENT) {
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_newentry_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->mkdir,
-					   &local->loc, st_mode);
-			if (!--call_count)
-				break;
-		}
-	}
+void
+afr_sh_purge_entry (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
 
-	return 0;
+        local = frame->local;
+        sh = &local->self_heal;
+        sh->post_remove_call = afr_sh_missing_entries_finish;
+
+        afr_sh_purge_entry_common (frame, this, afr_sh_purge_entry_condition);
 }
 
+void
+afr_sh_purge_stale_entry (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int             i = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
 
-static int
-sh_missing_entries_symlink (call_frame_t *frame, xlator_t *this,
-			    const char *link)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-	int              enoent_count = 0;
-	int              call_count = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sh->child_errno[i] == ENOENT)
-			enoent_count++;
-
-	call_count = enoent_count;
-	local->call_count = call_count;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"symlink %s -> %s on %d subvolumes",
-		local->loc.path, link, enoent_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i] == ENOENT) {
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_newentry_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->symlink,
-					   link, &local->loc);
-			if (!--call_count)
-				break;
-		}
-	}
+        sh->post_remove_call = afr_sh_purge_stale_entries_done;
 
-	return 0;
+        for (i = 0; i < priv->child_count; i++) {
+                if (afr_is_child_present (sh->fresh_children,
+                                          priv->child_count, i))
+                        continue;
+
+                if ((!local->child_up[i]) || sh->child_errno[i] != 0)
+                        continue;
+
+                GF_ASSERT (!uuid_is_null (sh->entrybuf.ia_gfid) ||
+                           uuid_is_null (sh->buf[i].ia_gfid));
+
+                if ((sh->entrybuf.ia_type != sh->buf[i].ia_type) ||
+                    (uuid_compare (sh->buf[i].ia_gfid,
+                                   sh->entrybuf.ia_gfid)))
+                        continue;
+
+                afr_children_add_child (sh->fresh_children, i,
+                                        priv->child_count);
+
+        }
+        afr_sh_purge_entry_common (frame, this,
+                                   afr_sh_purge_stale_entry_condition);
 }
 
+void
+afr_sh_save_child_iatts_from_policy (int32_t *children, struct iatt *bufs,
+                                     struct iatt *save,
+                                     unsigned int child_count)
+{
+        int             i = 0;
+        int             child = 0;
+        gf_boolean_t    saved = _gf_false;
+
+        GF_ASSERT (save);
+        //if iatt buf with gfid exists sets it
+        for (i = 0; i < child_count; i++) {
+                child = children[i];
+                if (child == -1)
+                        break;
+                *save = bufs[child];
+                saved = _gf_true;
+                if (!uuid_is_null (save->ia_gfid))
+                        break;
+        }
+        GF_ASSERT (saved);
+}
 
-static int
-sh_missing_entries_readlink_cbk (call_frame_t *frame, void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret, int32_t op_errno,
-				 const char *link)
+void
+afr_get_children_of_fresh_parent_dirs (afr_self_heal_t *sh,
+                                       unsigned int child_count)
 {
-	if (op_ret > 0)
-		sh_missing_entries_symlink (frame, this, link);
-	else
-		sh_missing_entries_finish (frame, this);
+        afr_children_intersection_get (sh->success_children,
+                                       sh->fresh_parent_dirs,
+                                       sh->sources, child_count);
+        afr_get_fresh_children (sh->success_children, sh->sources,
+                                sh->fresh_children, child_count);
+        memset (sh->sources, 0, sizeof (*sh->sources) * child_count);
+}
 
-	return 0;
+void
+afr_sh_children_lookup_done (call_frame_t *frame, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno)
+{
+        afr_local_t      *local = NULL;
+        afr_self_heal_t  *sh = NULL;
+        afr_private_t    *priv = NULL;
+        int32_t          fresh_child_enoents = 0;
+        int32_t          fresh_parent_count = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        if (op_ret < 0)
+                goto fail;
+        afr_get_children_of_fresh_parent_dirs (sh, priv->child_count);
+        fresh_parent_count = afr_get_children_count (sh->fresh_parent_dirs,
+                                                     priv->child_count);
+        //we need the enoent count of the subvols present in fresh_parent_dirs
+        fresh_child_enoents = afr_errno_count (sh->fresh_parent_dirs,
+                                               sh->child_errno,
+                                               priv->child_count, ENOENT);
+        if (fresh_child_enoents == fresh_parent_count) {
+                afr_sh_set_error (sh, ENOENT);
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                afr_sh_purge_entry (frame, this);
+        } else if (!afr_conflicting_iattrs (sh->buf, sh->fresh_children,
+                                            priv->child_count, local->loc.path,
+                                            this->name)) {
+                afr_sh_save_child_iatts_from_policy (sh->fresh_children,
+                                                     sh->buf, &sh->entrybuf,
+                                                     priv->child_count);
+                afr_update_gfid_from_iatts (sh->sh_gfid_req, sh->buf,
+                                            sh->fresh_children,
+                                            priv->child_count);
+                afr_sh_purge_stale_entry (frame, this);
+        } else {
+                op_errno = EIO;
+                afr_set_local_for_unhealable (local);
+                goto fail;
+        }
+
+        return;
+
+fail:
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        afr_sh_set_error (sh, op_errno);
+        afr_sh_missing_entries_finish (frame, this);
+        return;
 }
 
+static void
+afr_sh_find_fresh_parents (call_frame_t *frame, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno)
+{
+        afr_self_heal_t *sh  = NULL;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local = NULL;
+        int             enoent_count = 0;
+        int             nsources = 0;
+        int             source  = -1;
+        int32_t         subvol_status = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        if (op_ret < 0)
+                goto out;
+        enoent_count = afr_errno_count (NULL, sh->child_errno,
+                                        priv->child_count, ENOENT);
+        if (enoent_count > 0) {
+                gf_log (this->name, GF_LOG_INFO, "Parent dir missing for %s,"
+                        " in missing entry self-heal, aborting missing-entry "
+                        "self-heal",
+                        local->loc.path);
+                afr_sh_missing_entries_finish (frame, this);
+                return;
+        }
+
+        nsources = afr_build_sources (this, sh->xattr, sh->buf,
+                                      sh->pending_matrix, sh->sources,
+                                      sh->success_children,
+                                      AFR_ENTRY_TRANSACTION, &subvol_status,
+                                      _gf_true);
+        if ((subvol_status & ALL_FOOLS) ||
+            (subvol_status & SPLIT_BRAIN)) {
+                gf_log (this->name, GF_LOG_INFO, "%s: Performing conservative "
+                        "merge", sh->parent_loc.path);
+                afr_mark_success_children_sources (sh->sources,
+                                                   sh->success_children,
+                                                   priv->child_count);
+        } else if (nsources < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "No sources for dir "
+                        "of %s, in missing entry self-heal, aborting "
+                        "self-heal", local->loc.path);
+                op_errno = EIO;
+                goto out;
+        }
+
+        source = afr_sh_select_source (sh->sources, priv->child_count);
+        if (source == -1) {
+                GF_ASSERT (0);
+                gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
+                op_errno = EIO;
+                goto out;
+        }
+        afr_get_fresh_children (sh->success_children, sh->sources,
+                                sh->fresh_parent_dirs, priv->child_count);
+        afr_sh_common_lookup (frame, this, &local->loc,
+                              afr_sh_children_lookup_done, NULL, 0,
+                              NULL);
+        return;
 
-static int
-sh_missing_entries_readlink (call_frame_t *frame, xlator_t *this)
+out:
+        afr_sh_set_error (sh, op_errno);
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+	afr_sh_missing_entries_finish (frame, this);
+        return;
+}
+
+void
+afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
+        int             i = 0;
+
+        for (i = 0; i < child_count; i++) {
+                memset (&sh->buf[i], 0, sizeof (sh->buf[i]));
+                memset (&sh->parentbufs[i], 0, sizeof (sh->parentbufs[i]));
+                sh->child_errno[i] = 0;
+        }
+        memset (&sh->parentbuf, 0, sizeof (sh->parentbuf));
+        sh->success_count = 0;
+        afr_reset_children (sh->success_children, child_count);
+        afr_reset_children (sh->fresh_children, child_count);
+        afr_reset_xattr (sh->xattr, child_count);
+        loc_wipe (&sh->lookup_loc);
+}
 
+/* afr self-heal state will be lost if this call is made
+ * please check the afr_sh_common_reset that is called in this function
+ */
+int
+afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      afr_lookup_done_cbk_t lookup_done , uuid_t gfid,
+                      int32_t flags, dict_t *xdata)
+{
+        afr_local_t    *local = NULL;
+        int             i = 0;
+        int             call_count = 0;
+        afr_private_t  *priv = NULL;
+        dict_t         *xattr_req = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+        sh    = &local->self_heal;
+
+        call_count = afr_up_children_count (local->child_up, priv->child_count);
+
+        local->call_count = call_count;
+
+        xattr_req = dict_new();
+
+        if (xattr_req) {
+                afr_xattr_req_prepare (this, xattr_req, loc->path);
+                if (gfid) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "looking up %s with gfid: %s",
+                                loc->path, uuid_utoa (gfid));
+                        GF_ASSERT (!uuid_is_null (gfid));
+                        afr_set_dict_gfid (xattr_req, gfid);
+                }
+        }
+
+        afr_sh_common_reset (sh, priv->child_count);
+        sh->lookup_done = lookup_done;
+        loc_copy (&sh->lookup_loc, loc);
+        sh->lookup_flags = flags;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "looking up %s on subvolume %s",
+                                loc->path, priv->children[i]->name);
+
+                        STACK_WIND_COOKIE (frame,
+                                           afr_sh_common_lookup_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->lookup,
+                                           loc, xattr_req);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
+
+        if (xattr_req)
+                dict_unref (xattr_req);
+
+        return 0;
+}
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
 
-	STACK_WIND (frame, sh_missing_entries_readlink_cbk,
-		    priv->children[sh->source],
-		    priv->children[sh->source]->fops->readlink,
-		    &local->loc, 4096);
 
-	return 0;
+int
+afr_sh_post_nb_entrylk_missing_entry_sh_cbk (call_frame_t *frame,
+                                             xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        sh       = &local->self_heal;
+
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Non blocking entrylks failed.");
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                afr_sh_missing_entries_done (frame, this);
+        } else {
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Non blocking entrylks done. Proceeding to FOP");
+                afr_sh_common_lookup (frame, this, &sh->parent_loc,
+                                      afr_sh_find_fresh_parents,
+                                      NULL, AFR_LOOKUP_FAIL_CONFLICTS,
+                                      NULL);
+        }
+
+        return 0;
 }
 
-
-static int
-sh_missing_entries_create (call_frame_t *frame, xlator_t *this)
+int
+afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                char *base_name, afr_lock_cbk_t lock_cbk)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	int              type = 0;
-	int              i = 0;
-	afr_private_t   *priv = NULL;
-	int              enoent_count = 0;
-	int              govinda_gOvinda = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i]) {
-			if (sh->child_errno[i] == ENOENT)
-				enoent_count++;
-		} else {
-			if (type) {
-				if (type != (sh->buf[i].st_mode & S_IFMT))
-					govinda_gOvinda = 1;
-			} else {
-				sh->source = i;
-				type = sh->buf[i].st_mode & S_IFMT;
-			}
-		}
-	}
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
 
-	if (govinda_gOvinda) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"conflicing filetypes exist for path %s. returning.",
-			local->loc.path);
+        priv     = this->private;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
 
-		local->govinda_gOvinda = 1;
-		sh_missing_entries_finish (frame, this);
-		return 0;
-	}
+        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
+        int_lock->selfheal_lk_type    = AFR_ENTRY_SELF_HEAL_LK;
 
-	if (!type) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no source found for %s. all nodes down?. returning.",
-			local->loc.path);
-		/* subvolumes down and/or file does not exist */
-		sh_missing_entries_finish (frame, this);
-		return 0;
-	}
+        afr_set_lock_number (frame, this);
 
-	if (enoent_count == 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no missing files - %s. proceeding to metadata check",
-			local->loc.path);
-		/* proceed to next step - metadata self-heal */
-		sh_missing_entries_finish (frame, this);
-		return 0;
-	}
+        int_lock->lk_basename = base_name;
+        int_lock->lk_loc      = loc;
+        int_lock->lock_cbk    = lock_cbk;
+        int_lock->domain      = this->name;
 
-	switch (type) {
-	case S_IFSOCK:
-	case S_IFREG:
-	case S_IFBLK:
-	case S_IFCHR:
-	case S_IFIFO:
-		sh_missing_entries_mknod (frame, this);
-		break;
-	case S_IFLNK:
-		sh_missing_entries_readlink (frame, this);
-		break;
-	case S_IFDIR:
-		sh_missing_entries_mkdir (frame, this);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_ERROR,
-			"unknown file type: 0%o", type);
-		local->govinda_gOvinda = 1;
-		sh_missing_entries_finish (frame, this);
-	}
+        int_lock->lockee_count = 0;
+        afr_init_entry_lockee (&int_lock->lockee[0], local, loc,
+                               base_name, priv->child_count);
+        int_lock->lockee_count++;
+        afr_nonblocking_entrylk (frame, this);
 
-	return 0;
+        return 0;
 }
 
+static int
+afr_self_heal_parent_entrylk (call_frame_t *frame, xlator_t *this,
+                              afr_lock_cbk_t lock_cbk)
+{
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        int                 ret       = -1;
+        int32_t             op_errno  = 0;
+
+        local    = frame->local;
+        sh       = &local->self_heal;
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "attempting to recreate missing entries for path=%s",
+                local->loc.path);
+
+        ret = afr_build_parent_loc (&sh->parent_loc, &local->loc, &op_errno);
+        if (ret)
+                goto out;
+
+        afr_sh_entrylk (frame, this, &sh->parent_loc, NULL,
+                        lock_cbk);
+        return 0;
+out:
+        int_lock = &local->internal_lock;
+        int_lock->lock_op_ret = -1;
+        lock_cbk (frame, this);
+        return 0;
+}
 
 static int
-sh_missing_entries_lookup_cbk (call_frame_t *frame, void *cookie,
-			       xlator_t *this,
-			       int32_t op_ret, int32_t op_errno,
-			       inode_t *inode, struct stat *buf, dict_t *xattr)
-{
-	int              child_index = 0;
-	afr_local_t     *local = NULL;
-	int              call_count = 0;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"path %s on subvolume %s is of mode 0%o",
-				local->loc.path,
-				priv->children[child_index]->name,
-				buf->st_mode);
-
-			local->self_heal.buf[child_index] = *buf;
-		} else {
-			gf_log (this->name, GF_LOG_WARNING,
-				"path %s on subvolume %s => -1 (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-
-			local->self_heal.child_errno[child_index] = op_errno;
-		}
+afr_self_heal_missing_entries (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
 
-	}
-	UNLOCK (&frame->lock);
+        local = frame->local;
+        sh = &local->self_heal;
 
-	call_count = afr_frame_return (frame);
+        sh->sh_type_in_action  = AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY;
 
-	if (call_count == 0) {
-		sh_missing_entries_create (frame, this);
-	}
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
 
-	return 0;
+        afr_self_heal_parent_entrylk (frame, this,
+                                      afr_sh_post_nb_entrylk_missing_entry_sh_cbk);
+        return 0;
 }
 
+afr_local_t*
+afr_self_heal_local_init (afr_local_t *l, xlator_t *this)
+{
+        afr_private_t   *priv  = NULL;
+        afr_local_t     *lc    = NULL;
+        afr_self_heal_t *sh    = NULL;
+        afr_self_heal_t *shc   = NULL;
+        int             ret    = 0;
+
+        priv = this->private;
+
+        sh = &l->self_heal;
+
+        lc = mem_get0 (this->local_pool);
+        if (!lc)
+                goto out;
+
+        shc = &lc->self_heal;
+
+        shc->unwind = sh->unwind;
+        shc->gfid_sh_success_cbk = sh->gfid_sh_success_cbk;
+        shc->do_missing_entry_self_heal = sh->do_missing_entry_self_heal;
+        shc->do_gfid_self_heal = sh->do_gfid_self_heal;
+        shc->do_data_self_heal = sh->do_data_self_heal;
+        shc->do_metadata_self_heal = sh->do_metadata_self_heal;
+        shc->do_entry_self_heal = sh->do_entry_self_heal;
+        shc->force_confirm_spb = sh->force_confirm_spb;
+        shc->forced_merge = sh->forced_merge;
+        shc->background = sh->background;
+        shc->type = sh->type;
+        shc->data_sh_info = "";
+        shc->metadata_sh_info =  "";
+
+        uuid_copy (shc->sh_gfid_req, sh->sh_gfid_req);
+        if (l->loc.path) {
+                ret = loc_copy (&lc->loc, &l->loc);
+                if (ret < 0)
+                        goto out;
+        }
+
+        lc->child_up  = memdup (l->child_up,
+                                sizeof (*lc->child_up) * priv->child_count);
+        if (!lc->child_up) {
+                ret = -1;
+                goto out;
+        }
+
+        if (l->xattr_req)
+                lc->xattr_req = dict_ref (l->xattr_req);
+
+        if (l->cont.lookup.inode)
+                lc->cont.lookup.inode = inode_ref (l->cont.lookup.inode);
+        if (l->cont.lookup.xattr)
+                lc->cont.lookup.xattr = dict_ref (l->cont.lookup.xattr);
+
+        lc->internal_lock.locked_nodes =
+                             GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
+                                        priv->child_count, gf_afr_mt_char);
+        if (!lc->internal_lock.locked_nodes) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = afr_inodelk_init (&lc->internal_lock.inodelk[0],
+                                this->name, priv->child_count);
+        if (ret)
+                goto out;
 
-static int
-sh_missing_entries_lookup (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t    *local = NULL;
-	int             i = 0;
-	int             call_count = 0;
-	afr_private_t  *priv = NULL;
-	dict_t         *xattr_req = NULL;
-	int             ret = -1;
-
-	local = frame->local;
-	call_count = local->child_count;
-	priv = this->private;
-
-	local->call_count = call_count;
-	
-	xattr_req = dict_new();
-	
-	if (xattr_req)
-		ret = dict_set_uint64 (xattr_req, AFR_ENTRY_PENDING,
-				       priv->child_count * sizeof(int32_t));
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"looking up %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_lookup_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->lookup,
-					   &local->loc, xattr_req);
-
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	if (xattr_req)
-		dict_unref (xattr_req);
+out:
+        if (ret) {
+                afr_local_cleanup (lc, this);
+                lc = NULL;
+        }
+        return lc;
+}
 
-	return 0;
+int
+afr_self_heal_completion_cbk (call_frame_t *bgsh_frame, xlator_t *this)
+{
+        afr_private_t *   priv  = NULL;
+        afr_local_t *     local = NULL;
+        afr_self_heal_t * sh    = NULL;
+        afr_local_t *     orig_frame_local = NULL;
+        afr_self_heal_t * orig_frame_sh = NULL;
+        char              sh_type_str[256] = {0,};
+        gf_loglevel_t     loglevel = 0;
+
+        priv  = this->private;
+        local = bgsh_frame->local;
+        sh    = &local->self_heal;
+
+        if (local->unhealable) {
+                afr_set_split_brain (this, sh->inode, SPB, SPB);
+        }
+
+        afr_self_heal_type_str_get (sh, sh_type_str,
+                                    sizeof(sh_type_str));
+        if (is_self_heal_failed (sh, AFR_CHECK_ALL) && !priv->shd.iamshd) {
+                loglevel = GF_LOG_ERROR;
+        } else if (!is_self_heal_failed (sh, AFR_CHECK_ALL)) {
+                loglevel = GF_LOG_INFO;
+        } else {
+                loglevel = GF_LOG_DEBUG;
+        }
+
+        afr_log_self_heal_completion_status (local, loglevel);
+
+        FRAME_SU_UNDO (bgsh_frame, afr_local_t);
+
+        if (!sh->unwound && sh->unwind) {
+                orig_frame_local = sh->orig_frame->local;
+                orig_frame_sh = &orig_frame_local->self_heal;
+                orig_frame_sh->actual_sh_started = _gf_true;
+                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
+                            is_self_heal_failed (sh, AFR_CHECK_ALL));
+        }
+
+        if (sh->background) {
+                LOCK (&priv->lock);
+                {
+                        priv->background_self_heals_started--;
+                }
+                UNLOCK (&priv->lock);
+        }
+
+        AFR_STACK_DESTROY (bgsh_frame);
+
+        return 0;
 }
 
+int
+afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int32_t          op_errno = 0;
+        int              ret = 0;
+        afr_self_heal_t *orig_sh = NULL;
+        call_frame_t    *sh_frame = NULL;
+        afr_local_t     *sh_local = NULL;
+        loc_t           *loc   = NULL;
+
+        local = frame->local;
+        orig_sh = &local->self_heal;
+        priv  = this->private;
+
+        GF_ASSERT (local->loc.path);
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "performing self heal on %s (metadata=%d data=%d entry=%d)",
+                local->loc.path,
+                local->self_heal.do_metadata_self_heal,
+                local->self_heal.do_data_self_heal,
+                local->self_heal.do_entry_self_heal);
+
+        op_errno        = ENOMEM;
+        sh_frame        = copy_frame (frame);
+        if (!sh_frame)
+                goto out;
+        afr_set_lk_owner (sh_frame, this, sh_frame->root);
+        afr_set_low_priority (sh_frame);
+
+        sh_local        = afr_self_heal_local_init (local, this);
+        if (!sh_local)
+                goto out;
+        sh_frame->local = sh_local;
+        sh              = &sh_local->self_heal;
+
+        sh->inode       = inode_ref (inode);
+        sh->orig_frame  = frame;
+
+        sh->completion_cbk = afr_self_heal_completion_cbk;
+
+        sh->success = GF_CALLOC (priv->child_count, sizeof (*sh->success),
+                                 gf_afr_mt_char);
+        if (!sh->success)
+                goto out;
+        sh->sources = GF_CALLOC (sizeof (*sh->sources), priv->child_count,
+                                 gf_afr_mt_int);
+        if (!sh->sources)
+                goto out;
+        sh->locked_nodes = GF_CALLOC (sizeof (*sh->locked_nodes),
+                                      priv->child_count,
+                                      gf_afr_mt_int);
+        if (!sh->locked_nodes)
+                goto out;
+
+        sh->pending_matrix = afr_matrix_create (priv->child_count,
+                                                priv->child_count);
+        if (!sh->pending_matrix)
+                goto out;
+
+        sh->delta_matrix = afr_matrix_create (priv->child_count,
+                                              priv->child_count);
+        if (!sh->delta_matrix)
+                goto out;
+
+        sh->fresh_parent_dirs = afr_children_create (priv->child_count);
+        if (!sh->fresh_parent_dirs)
+                goto out;
+        ret = afr_sh_common_create (sh, priv->child_count);
+        if (ret) {
+                op_errno = -ret;
+                goto out;
+        }
+
+        if (local->self_heal.background) {
+                LOCK (&priv->lock);
+                {
+                        if (priv->background_self_heals_started
+                            < priv->background_self_heal_count) {
+                                priv->background_self_heals_started++;
+
+
+                        } else {
+                                local->self_heal.background = _gf_false;
+                                sh->background = _gf_false;
+                        }
+                }
+                UNLOCK (&priv->lock);
+        }
+
+        if (!local->loc.parent) {
+                sh->do_missing_entry_self_heal = _gf_false;
+                sh->do_gfid_self_heal = _gf_false;
+        }
+
+        sh->sh_type_in_action = AFR_SELF_HEAL_INVALID;
+
+        FRAME_SU_DO (sh_frame, afr_local_t);
+        if (sh->do_missing_entry_self_heal || sh->do_gfid_self_heal) {
+                afr_self_heal_missing_entries (sh_frame, this);
+        } else {
+                loc = &sh_local->loc;
+                if (uuid_is_null (loc->inode->gfid) && uuid_is_null (loc->gfid)) {
+                        if (!uuid_is_null (inode->gfid))
+                                GF_ASSERT (!uuid_compare (inode->gfid,
+                                           sh->sh_gfid_req));
+                        uuid_copy (loc->gfid, sh->sh_gfid_req);
+                }
+                gf_log (this->name, GF_LOG_TRACE,
+                        "proceeding to metadata check on %s",
+                        local->loc.path);
+
+                afr_sh_missing_entries_done (sh_frame, this);
+        }
+        op_errno = 0;
 
-static int
-sh_missing_entries_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	int              call_count = 0;
-	int              child_index = (long) cookie;
-
-
-	local = frame->local;
-	sh    = &local->self_heal;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			sh->op_failed = 1;
-
-			gf_log (this->name,
-				(op_errno == EAGAIN ? GF_LOG_DEBUG : GF_LOG_ERROR),
-				"locking inode of %s on child %d failed: %s",
-				local->loc.path, child_index,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"inode of %s on child %d locked",
-				local->loc.path, child_index);
-		}
-	}
-	UNLOCK (&frame->lock);
+out:
+        if (op_errno) {
+                orig_sh->unwind (frame, this, -1, op_errno, 1);
+                if (sh_frame)
+                        AFR_STACK_DESTROY (sh_frame);
+        }
+        return 0;
+}
 
-	call_count = afr_frame_return (frame);
+void
+afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
+                            size_t size)
+{
+        GF_ASSERT (str && (size > strlen (" missing-entry gfid "
+                                          "meta-data data entry")));
 
-	if (call_count == 0) {
-		if (sh->op_failed == 1) {
-			sh_missing_entries_finish (frame, this);
-			return 0;
-		}
+        if (self_heal_p->do_metadata_self_heal) {
+                snprintf (str, size, " meta-data");
+        }
 
-		sh_missing_entries_lookup (frame, this);
-	}
+        if (self_heal_p->do_data_self_heal) {
+                snprintf (str + strlen(str), size - strlen(str), " data");
+        }
 
-	return 0;
+        if (self_heal_p->do_entry_self_heal) {
+                snprintf (str + strlen(str), size - strlen(str), " entry");
+        }
+
+        if (self_heal_p->do_missing_entry_self_heal) {
+                snprintf (str + strlen(str), size - strlen(str),
+                         " missing-entry");
+        }
+
+        if (self_heal_p->do_gfid_self_heal) {
+                snprintf (str + strlen(str), size - strlen(str), " gfid");
+        }
 }
 
+afr_self_heal_type
+afr_self_heal_type_for_transaction (afr_transaction_type type)
+{
+        afr_self_heal_type sh_type = AFR_SELF_HEAL_INVALID;
+
+        switch (type) {
+        case AFR_DATA_TRANSACTION:
+                sh_type = AFR_SELF_HEAL_DATA;
+                break;
+        case AFR_METADATA_TRANSACTION:
+                sh_type = AFR_SELF_HEAL_METADATA;
+                break;
+        case AFR_ENTRY_TRANSACTION:
+                sh_type = AFR_SELF_HEAL_ENTRY;
+                break;
+        case AFR_ENTRY_RENAME_TRANSACTION:
+                GF_ASSERT (0);
+                break;
+        }
+        return sh_type;
+}
 
-static int
-afr_self_heal_missing_entries (call_frame_t *frame, xlator_t *this)
+int
+afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-	int              call_count = 0;
+        int   ret = -1;
+        uuid_t pargfid = {0};
 
+        if (!child)
+                goto out;
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        if (!uuid_is_null (parent->inode->gfid))
+                uuid_copy (pargfid, parent->inode->gfid);
+        else if (!uuid_is_null (parent->gfid))
+                uuid_copy (pargfid, parent->gfid);
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"attempting to recreate missing entries for path=%s",
-		local->loc.path);
+        if (uuid_is_null (pargfid))
+                goto out;
 
-	afr_build_parent_loc (&sh->parent_loc, &local->loc);
+        if (strcmp (parent->path, "/") == 0)
+                ret = gf_asprintf ((char **)&child->path, "/%s", name);
+        else
+                ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path,
+                                   name);
 
-	call_count = local->child_count;
+        if (-1 == ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "asprintf failed while setting child path");
+        }
 
-	local->call_count = call_count;
+        child->name = strrchr (child->path, '/');
+        if (child->name)
+                child->name++;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, sh_missing_entries_lk_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->entrylk,
-				    &sh->parent_loc, local->loc.name,
-				    ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
-			if (!--call_count)
-				break;
-		}
- 	}
+        child->parent = inode_ref (parent->inode);
+        child->inode = inode_new (parent->inode->table);
+        uuid_copy (child->pargfid, pargfid);
 
-	return 0;
+        if (!child->inode) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if ((ret == -1) && child)
+                loc_wipe (child);
+
+        return ret;
 }
 
+int
+afr_sh_erase_pending (call_frame_t *frame, xlator_t *this,
+                      afr_transaction_type type, afr_fxattrop_cbk_t cbk,
+                      int (*finish)(call_frame_t *frame, xlator_t *this))
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              call_count = 0;
+        int              i = 0;
+        dict_t          **erase_xattr = NULL;
+        int             ret = -1;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix,
+                                 sh->success, priv->child_count, type);
+
+        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
+                                 gf_afr_mt_dict_t);
+        if (!erase_xattr)
+                goto out;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (sh->xattr[i]) {
+                        call_count++;
+                        erase_xattr[i] = dict_new ();
+                        if (!erase_xattr[i])
+                                goto out;
+                }
+        }
+
+        afr_sh_delta_to_xattr (this, sh->delta_matrix, erase_xattr,
+                               priv->child_count, type);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Delta matrix for: %s",
+                lkowner_utoa (&frame->root->lk_owner));
+        afr_sh_print_pending_matrix (sh->delta_matrix, this);
+        local->call_count = call_count;
+        if (call_count == 0) {
+                ret = 0;
+                finish (frame, this);
+                goto out;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!erase_xattr[i])
+                        continue;
+
+                if (sh->healing_fd) {//true for ENTRY, reg file DATA transaction
+                        STACK_WIND_COOKIE (frame, cbk, (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fxattrop,
+                                           sh->healing_fd,
+                                           GF_XATTROP_ADD_ARRAY, erase_xattr[i],
+                                           NULL);
+                } else {
+                        STACK_WIND_COOKIE (frame, cbk, (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->xattrop,
+                                           &local->loc,
+                                           GF_XATTROP_ADD_ARRAY, erase_xattr[i],
+                                           NULL);
+                }
+        }
+
+        ret = 0;
+out:
+        if (erase_xattr) {
+                for (i = 0; i < priv->child_count; i++) {
+                        if (erase_xattr[i]) {
+                                dict_unref (erase_xattr[i]);
+                        }
+                }
+        }
+
+        GF_FREE (erase_xattr);
+
+        if (ret < 0) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                finish (frame, this);
+        }
+
+        return 0;
+}
+
+void
+afr_set_self_heal_status(afr_self_heal_t *sh, afr_self_heal_status status)
+{
+        xlator_t                *this = NULL;
+        afr_sh_status_for_all_type *sh_status = &(sh->afr_all_sh_status);
+        afr_self_heal_type  sh_type_in_action = sh->sh_type_in_action;
+        this = THIS;
+
+        if (!sh) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Null self heal"
+                                  "Structure");
+                goto out;
+        }
+
+        switch (sh_type_in_action) {
+                case AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY:
+                       sh_status->gfid_or_missing_entry_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_METADATA:
+                        sh_status->metadata_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_DATA:
+                        sh_status->data_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_ENTRY:
+                        sh_status->entry_self_heal = status;
+                        break;
+                case AFR_SELF_HEAL_INVALID:
+                        gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid"
+                                          "self heal type in action");
+                        break;
+        }
+out:
+        return;
+}
+
+void
+afr_set_local_for_unhealable (afr_local_t *local)
+{
+        afr_self_heal_t  *sh = NULL;
+
+        sh = &local->self_heal;
+
+        local->unhealable = 1;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+}
 
 int
-afr_self_heal (call_frame_t *frame, xlator_t *this,
-	       int (*completion_cbk) (call_frame_t *, xlator_t *))
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"performing self heal on %s (metadata=%d data=%d entry=%d)",
-		local->loc.path,
-		local->need_metadata_self_heal,
-		local->need_data_self_heal,
-		local->need_entry_self_heal);
-
-	sh->completion_cbk = completion_cbk;
-
-	sh->buf = CALLOC (priv->child_count, sizeof (struct stat));
-	sh->child_errno = CALLOC (priv->child_count, sizeof (int));
-	sh->success = CALLOC (priv->child_count, sizeof (int));
-	sh->xattr = CALLOC (priv->child_count, sizeof (dict_t *));
-	sh->sources = CALLOC (sizeof (*sh->sources), priv->child_count);
-
-	sh->pending_matrix = CALLOC (sizeof (int32_t *), priv->child_count);
-	for (i = 0; i < priv->child_count; i++) {
-		sh->pending_matrix[i] = CALLOC (sizeof (int32_t),
-						priv->child_count);
-	}
+is_self_heal_failed (afr_self_heal_t *sh, afr_sh_fail_check_type type)
+{
+        afr_sh_status_for_all_type      sh_status = sh->afr_all_sh_status;
+        afr_self_heal_type   sh_type_in_action =  AFR_SELF_HEAL_INVALID;
+        afr_self_heal_status    status = AFR_SELF_HEAL_FAILED;
+        xlator_t                *this = NULL;
+        int                     sh_failed = 0;
+
+        this = THIS;
+
+        if (!sh) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Null self heal "
+                                  "structure");
+                sh_failed = 1;
+                goto out;
+        }
+
+        if (type == AFR_CHECK_ALL) {
+                if ((sh_status.gfid_or_missing_entry_self_heal == AFR_SELF_HEAL_FAILED)
+                    || (sh_status.metadata_self_heal == AFR_SELF_HEAL_FAILED)
+                    || (sh_status.data_self_heal == AFR_SELF_HEAL_FAILED)
+                    || (sh_status.entry_self_heal == AFR_SELF_HEAL_FAILED))
+                sh_failed = 1;
+        } else if (type == AFR_CHECK_SPECIFIC) {
+                sh_type_in_action = sh->sh_type_in_action;
+                switch (sh_type_in_action) {
+                        case AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY:
+                             status = sh_status.gfid_or_missing_entry_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_METADATA:
+                                status = sh_status.metadata_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_ENTRY:
+                                status = sh_status.entry_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_DATA:
+                                status = sh_status.data_self_heal;
+                                break;
+                        case AFR_SELF_HEAL_INVALID:
+                                status = AFR_SELF_HEAL_NOT_ATTEMPTED;
+                                break;
+                }
+                if (status == AFR_SELF_HEAL_FAILED)
+                        sh_failed = 1;
+
+        }
 
-	sh->delta_matrix = CALLOC (sizeof (int32_t *), priv->child_count);
-	for (i = 0; i < priv->child_count; i++) {
-		sh->delta_matrix[i] = CALLOC (sizeof (int32_t),
-					      priv->child_count);
-	}
+out:
+        return sh_failed;
+}
 
-	if (local->success_count && local->enoent_count) {
-		afr_self_heal_missing_entries (frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"proceeding to metadata check on %s",
-			local->loc.path);
-		afr_sh_missing_entries_done (frame, this);
-	}
+char *
+get_sh_completion_status (afr_self_heal_status status)
+{
 
-	return 0;
+        char *not_attempted       = " is not attempted";
+        char *failed              = " failed";
+        char *started             = " is started";
+        char *sync_begin          = " is successfully completed";
+        char *result              = " has unknown status";
+
+        switch (status)
+        {
+                case AFR_SELF_HEAL_NOT_ATTEMPTED:
+                        result = not_attempted;
+                        break;
+                case AFR_SELF_HEAL_FAILED:
+                        result = failed;
+                        break;
+                case AFR_SELF_HEAL_STARTED:
+                        result = started;
+                        break;
+                case AFR_SELF_HEAL_SYNC_BEGIN:
+                        result = sync_begin;
+                        break;
+        }
+
+        return result;
+
+}
+
+void
+afr_log_self_heal_completion_status (afr_local_t *local, gf_loglevel_t loglvl)
+{
+
+        char sh_log[4096]              = {0};
+        afr_self_heal_t *sh            = &local->self_heal;
+        afr_sh_status_for_all_type   all_status = sh->afr_all_sh_status;
+        xlator_t      *this            = NULL;
+        size_t        off              = 0;
+        int           data_sh          = 0;
+        int           metadata_sh      = 0;
+        int           print_log        = 0;
+
+        this = THIS;
+
+        ADD_FMT_STRING (sh_log, off, "gfid or missing entry",
+                        all_status.gfid_or_missing_entry_self_heal, print_log);
+        ADD_FMT_STRING_SYNC (sh_log, off, "metadata",
+                             all_status.metadata_self_heal, print_log);
+        if (sh->background) {
+                ADD_FMT_STRING_SYNC (sh_log, off, "backgroung data",
+                                all_status.data_self_heal, print_log);
+        } else {
+                ADD_FMT_STRING_SYNC (sh_log, off, "foreground data",
+                                all_status.data_self_heal, print_log);
+        }
+        ADD_FMT_STRING_SYNC (sh_log, off, "entry", all_status.entry_self_heal,
+                             print_log);
+
+        if (AFR_SELF_HEAL_SYNC_BEGIN == all_status.data_self_heal &&
+	    strcmp (sh->data_sh_info, "") && sh->data_sh_info )
+                data_sh = 1;
+        if (AFR_SELF_HEAL_SYNC_BEGIN == all_status.metadata_self_heal &&
+	    strcmp (sh->metadata_sh_info, "") && sh->metadata_sh_info)
+                metadata_sh = 1;
+
+        if (!print_log)
+                return;
+
+        gf_log (this->name, loglvl, "%s %s %s on %s", sh_log,
+                ((data_sh == 1) ? sh->data_sh_info : ""),
+                ((metadata_sh == 1) ? sh->metadata_sh_info : ""),
+                local->loc.path);
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.h b/xlators/cluster/afr/src/afr-self-heal-common.h
index 9dd597f07..473264776 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.h
+++ b/xlators/cluster/afr/src/afr-self-heal-common.h
@@ -1,66 +1,144 @@
 /*
-  Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEAL_COMMON_H__
 #define __AFR_SELF_HEAL_COMMON_H__
 
-#define FILE_HAS_HOLES(buf) (((buf)->st_size) > ((buf)->st_blocks * 512))
+#define FILE_HAS_HOLES(buf) (((buf)->ia_size) > ((buf)->ia_blocks * 512))
+#define AFR_SH_MIN_PARTICIPANTS 2
+
+typedef enum {
+        AFR_LOOKUP_FAIL_CONFLICTS = 1,
+        AFR_LOOKUP_FAIL_MISSING_GFIDS = 2,
+} afr_lookup_flags_t;
 
 int
 afr_sh_select_source (int sources[], int child_count);
 
 int
-afr_sh_sink_count (int sources[], int child_count);
+afr_sh_source_count (int sources[], int child_count);
+
+void
+afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this);
+
+void
+afr_sh_print_split_brain_log (int32_t *pending_matrix[], xlator_t *this,
+                              const char *loc);
 
 int
-afr_sh_source_count (int sources[], int child_count);
+afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
+                          unsigned char *ignorant_subvols,
+                          dict_t *xattr[], afr_transaction_type type,
+                          size_t child_count);
+
+void
+afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
+                         int32_t *delta_matrix[], unsigned char success[],
+                         int child_count, afr_transaction_type type);
 
 int
-afr_sh_supress_errenous_children (int sources[], int child_errno[],
-				  int child_count);
+afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
+                  struct iatt *bufs, afr_self_heal_type type,
+                  int32_t *success_children, int32_t *subvol_status);
 
 int
-afr_sh_supress_empty_children (int sources[], dict_t *xattr[],
-			       struct stat *buf,
-			       int child_count, const char *key);
+afr_sh_delta_to_xattr (xlator_t *this,
+                       int32_t *delta_matrix[], dict_t *xattr[],
+		       int child_count, afr_transaction_type type);
 
 void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this);
+afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
+                            size_t size);
+
+afr_self_heal_type
+afr_self_heal_type_for_transaction (afr_transaction_type type);
 
+int
+afr_build_sources (xlator_t *this, dict_t **xattr, struct iatt *bufs,
+                   int32_t **pending_matrix, int32_t *sources,
+                   int32_t *success_children, afr_transaction_type type,
+                   int32_t *subvol_status, gf_boolean_t ignore_ignorant);
 void
-afr_sh_build_pending_matrix (int32_t *pending_matrix[], dict_t *xattr[],
-			     int child_count, const char *key);
+afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count);
 
 void
-afr_sh_pending_to_delta (int32_t *pending_matrix[], int32_t *delta_matrix[],
-			 int32_t success[], int child_count);
+afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
+                                   xlator_t *this,
+                                   int32_t op_ret, int32_t op_errno,
+                                   inode_t *inode, struct iatt *buf,
+                                   dict_t *xattr, struct iatt *postparent,
+                                   loc_t *loc);
 
 int
-afr_sh_mark_sources (int32_t *pending_matrix[], int sources[],
-		     int child_count);
+afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      afr_lookup_done_cbk_t lookup_cbk, uuid_t uuid,
+                      int32_t flags, dict_t *xdata);
+int
+afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
+                             int active_src, struct iatt *buf,
+                             struct iatt *parentbuf);
+int
+afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                char *base_name, afr_lock_cbk_t lock_cbk);
+int
+afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
+                             int child_index);
+int
+afr_sh_data_unlock (call_frame_t *frame, xlator_t *this, char *dom,
+                    afr_lock_cbk_t lock_cbk);
+afr_local_t *
+afr_self_heal_local_init (afr_local_t *l, xlator_t *this);
+int
+afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
+                  off_t start, off_t len, gf_boolean_t block, char *dom,
+                  afr_lock_cbk_t success_handler,
+                  afr_lock_cbk_t failure_handler);
+void
+afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno);
+void
+afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this);
+typedef int
+(*afr_fxattrop_cbk_t) (call_frame_t *frame, void *cookie,
+                       xlator_t *this, int32_t op_ret, int32_t op_errno,
+                       dict_t *xattr, dict_t *xdata);
+int
+afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
+int
+afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
+                          int active_source, call_frame_t **impunge_frame);
+void
+afr_sh_reset (call_frame_t *frame, xlator_t *this);
 
+void
+afr_children_intersection_get (int32_t *set1, int32_t *set2,
+                               int *intersection, unsigned int child_count);
 int
-afr_sh_delta_to_xattr (int32_t *delta_matrix[], dict_t *xattr[],
-		       int child_count, const char *key);
+afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
+                                 struct iatt *bufs);
+int
+afr_sh_erase_pending (call_frame_t *frame, xlator_t *this,
+                      afr_transaction_type type, afr_fxattrop_cbk_t cbk,
+                      int (*finish)(call_frame_t *frame, xlator_t *this));
+
+void
+afr_set_local_for_unhealable (afr_local_t *local);
 
 int
-afr_sh_is_matrix_zero (int32_t *pending_matrix[], int child_count);
+is_self_heal_failed (afr_self_heal_t *sh, afr_sh_fail_check_type type);
+
+void
+afr_set_self_heal_status (afr_self_heal_t *sh, afr_self_heal_status status);
 
+void
+afr_log_self_heal_completion_status (afr_local_t *local, gf_loglevel_t  logl);
 
+char*
+afr_get_pending_matrix_str (int32_t *pending_matrix[], xlator_t *this);
 #endif /* __AFR_SELF_HEAL_COMMON_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 3a48da485..9de26ee56 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -47,984 +38,1717 @@
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
 #include "afr-self-heal-common.h"
-
-
+#include "afr-self-heal-algorithm.h"
 
 int
-afr_sh_data_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+afr_sh_data_fail (call_frame_t *frame, xlator_t *this);
 
-	/* 
-	   TODO: cleanup sh->* 
-	 */
+static inline gf_boolean_t
+afr_sh_data_proceed (unsigned int success_count)
+{
+        return (success_count >= AFR_SH_MIN_PARTICIPANTS);
+}
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"self heal of %s completed",
-		local->loc.path);
+extern int
+sh_loop_finish (call_frame_t *loop_frame, xlator_t *this);
 
-	sh->completion_cbk (frame, this);
+int
+afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this);
 
-	return 0;
-}
+int
+afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this);
 
+int
+afr_sh_data_finish (call_frame_t *frame, xlator_t *this);
 
 int
-afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+afr_sh_data_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv  = NULL;
-	afr_self_heal_t *sh = NULL;
-	int              call_count = 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
 
-	LOCK (&frame->lock);
-	{
-	}
-	UNLOCK (&frame->lock);
+        sh->completion_cbk (frame, this);
 
-	call_count = afr_frame_return (frame);
+        return 0;
+}
 
-	if (call_count == 0) {
-		fd_unref (sh->healing_fd);
-		sh->healing_fd = NULL;
-		afr_sh_data_done (frame, this);
-	}
 
-	return 0;
+int
+afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_local_t   *local       = NULL;
+        afr_private_t *priv        = NULL;
+        int            call_count  = 0;
+        int            child_index = (long) cookie;
+
+        local = frame->local;
+        priv = this->private;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "flush failed on %s on subvolume %s: %s",
+                                local->loc.path, priv->children[child_index]->name,
+                                strerror (op_errno));
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                afr_sh_data_done (frame, this);
+        }
+
+        return 0;
 }
 
-
 int
 afr_sh_data_close (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              i = 0;
-	int              call_count = 0;
-
+        afr_local_t     *local      = NULL;
+        afr_private_t   *priv       = NULL;
+        afr_self_heal_t *sh         = NULL;
+        int              i          = 0;
+        int              call_count = 0;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+        priv  = this->private;
+
+        if (!sh->healing_fd) {
+                //This happens when file is non-reg
+                afr_sh_data_done (frame, this);
+                return 0;
+        }
+        call_count        = afr_set_elem_count_get (sh->success,
+                                                    priv->child_count);
+        local->call_count = call_count;
+
+        if (call_count == 0) {
+                afr_sh_data_done (frame, this);
+                return 0;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!sh->success[i])
+                        continue;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "closing fd of %s on %s",
+                        local->loc.path, priv->children[i]->name);
+
+                STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
+                                   (void *) (long) i,
+                                   priv->children[i],
+                                   priv->children[i]->fops->flush,
+                                   sh->healing_fd, NULL);
+
+                if (!--call_count)
+                        break;
+        }
+
+        return 0;
+}
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+int
+afr_sh_dom_unlock (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh    = NULL;
+        afr_private_t   *priv  = NULL;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+        priv  = this->private;
+
+        if (sh->sh_dom_lock_held)
+                afr_sh_data_unlock (frame, this, priv->sh_domain,
+                                    afr_sh_data_close);
+        else
+                afr_sh_data_close (frame, this);
+        return 0;
+}
 
-	if (!sh->healing_fd) {
-		afr_sh_data_done (frame, this);
-		return 0;
-	}
+int
+afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                         struct iatt *statpost, dict_t *xdata)
+{
 
-	call_count = sh->active_sinks + 1;
-	local->call_count = call_count;
+        afr_local_t   *local       = NULL;
+        afr_private_t *priv        = NULL;
+        int            call_count  = 0;
+        int            child_index = (long) cookie;
 
+        local = frame->local;
+        priv = this->private;
 
-	/* closed source */
-	gf_log (this->name, GF_LOG_DEBUG,
-		"closing fd of %s on %s",
-		local->loc.path, priv->children[sh->source]->name);
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "setattr failed on %s on subvolume %s: %s",
+                                local->loc.path, priv->children[child_index]->name,
+                                strerror (op_errno));
+                }
+        }
+        UNLOCK (&frame->lock);
 
-	STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
-			   (void *) (long) sh->source,
-			   priv->children[sh->source],
-			   priv->children[sh->source]->fops->flush,
-			   sh->healing_fd);
-	call_count--;
+        call_count = afr_frame_return (frame);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] || !local->child_up[i])
-			continue;
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"closing fd of %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->flush,
-				   sh->healing_fd);
-		if (!--call_count)
-			break;
-	}
+        if (call_count == 0) {
+                afr_sh_data_finish (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
+int
+afr_sh_data_setattr (call_frame_t *frame, xlator_t *this, struct iatt* stbuf)
+{
+        afr_local_t     *local      = NULL;
+        afr_private_t   *priv       = NULL;
+        afr_self_heal_t *sh         = NULL;
+        int              i          = 0;
+        int              call_count = 0;
+        int32_t          valid      = 0;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+        priv  = this->private;
+
+        valid = (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
+
+        call_count        = afr_set_elem_count_get (sh->success,
+                                                    priv->child_count);
+        local->call_count = call_count;
+
+        if (call_count == 0) {
+                GF_ASSERT (0);
+                afr_sh_data_finish (frame, this);
+                return 0;
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!sh->success[i])
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_sh_data_setattr_cbk,
+                                   (void *) (long) i,
+                                   priv->children[i],
+                                   priv->children[i]->fops->setattr,
+                                   &local->loc, stbuf, valid, NULL);
+
+                if (!--call_count)
+                        break;
+        }
+
+        return 0;
+}
 
 int
-afr_sh_data_unlck_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+afr_sh_data_setattr_fstat_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               struct iatt *buf, dict_t *xdata)
 {
-	afr_local_t * local = NULL;
-	int           call_count = 0;
-	int           child_index = (long) cookie;
-
-	
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"locking inode of %s on child %d failed: %s",
-				local->loc.path, child_index,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"inode of %s on child %d locked",
-				local->loc.path, child_index);
-		}
-	}
-	UNLOCK (&frame->lock);
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        int child_index = (long) cookie;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        GF_ASSERT (sh->source == child_index);
+        if (op_ret != -1) {
+                sh->buf[child_index] = *buf;
+                afr_sh_data_setattr (frame, this, buf);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
+                        "time-stamps after self-heal", local->loc.path);
+                afr_sh_data_fail (frame, this);
+        }
+
+        return 0;
+}
 
-	call_count = afr_frame_return (frame);
+/*
+ * If there are any writes after the self-heal is triggered then the
+ * stbuf stored in local->self_heal.buf[] will be invalid so we do one more
+ * stat on the source and then set the [am]times
+ */
+int
+afr_sh_set_timestamps (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local      = NULL;
+        afr_private_t   *priv       = NULL;
+        afr_self_heal_t *sh         = NULL;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+        priv  = this->private;
+
+        STACK_WIND_COOKIE (frame, afr_sh_data_setattr_fstat_cbk,
+                           (void *) (long) sh->source,
+                           priv->children[sh->source],
+                           priv->children[sh->source]->fops->fstat,
+                           sh->healing_fd, NULL);
+        return 0;
+}
 
-	if (call_count == 0) {
-		afr_sh_data_close (frame, this);
-	}
+//Fun fact, lock_cbk is being used for both lock & unlock
+int
+afr_sh_data_unlock (call_frame_t *frame, xlator_t *this, char *dom,
+                    afr_lock_cbk_t lock_cbk)
+{
+        afr_local_t         *local    = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_self_heal_t     *sh       = NULL;
+        afr_private_t       *priv     = NULL;
+        int                 ret       = 0;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        sh       = &local->self_heal;
+        priv     = this->private;
+
+        if (strcmp (dom, this->name) == 0) {
+                sh->data_lock_held = _gf_false;
+        } else if (strcmp (dom, priv->sh_domain) == 0) {
+                sh->sh_dom_lock_held = _gf_false;
+        } else {
+                ret = -1;
+                goto out;
+        }
+        int_lock->lock_cbk = lock_cbk;
+        int_lock->domain = dom;
+        afr_unlock (frame, this);
 
-	return 0;
+out:
+        if (ret) {
+                int_lock->lock_op_ret = -1;
+                int_lock->lock_cbk (frame, this);
+        }
+        return 0;
 }
 
-
 int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this)
+afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
 {
-	struct flock flock;			
-	int i = 0;				
-	int call_count = 0;		     
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh    = NULL;
 
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t * sh  = NULL;
+        local = frame->local;
+        sh = &local->self_heal;
 
+        gf_log (this->name, GF_LOG_DEBUG,
+                "finishing data selfheal of %s", local->loc.path);
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        if (sh->data_lock_held)
+                afr_sh_data_unlock (frame, this, this->name, afr_sh_dom_unlock);
+        else
+                afr_sh_dom_unlock (frame, this);
 
-	call_count = local->child_count;
+        return 0;
+}
 
-	local->call_count = call_count;		
+int
+afr_sh_data_fail (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_self_heal_t *sh = NULL;
 
-	flock.l_start = 0;
-	flock.l_len   = 0;
-	flock.l_type  = F_UNLCK;
+        local = frame->local;
+        sh = &local->self_heal;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"unlocking %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame, afr_sh_data_unlck_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->inodelk,
-					   &local->loc, F_SETLK, &flock); 
-			if (!--call_count)
-				break;
-		}
-	}
+        gf_log (this->name, GF_LOG_DEBUG,
+                "finishing failed data selfheal of %s", local->loc.path);
 
-	return 0;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        afr_sh_data_finish (frame, this);
+        return 0;
 }
 
-
 int
-afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
+afr_sh_data_erase_pending_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret,
+                               int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-	afr_local_t   *local = NULL;
-
-	local = frame->local;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"finishing data selfheal of %s", local->loc.path);
-
-	afr_sh_data_unlock (frame, this);
+        int             call_count = 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int32_t         child_index = (long) cookie;
+
+        priv  = this->private;
+        local = frame->local;
+        sh    = &local->self_heal;
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Erasing of pending change "
+                        "log failed on %s for subvol %s, reason: %s",
+                        local->loc.path, priv->children[child_index]->name,
+                        strerror (op_errno));
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        }
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                        if (sh->old_loop_frame)
+                                sh_loop_finish (sh->old_loop_frame, this);
+                        sh->old_loop_frame = NULL;
+                        afr_sh_data_fail (frame, this);
+                        goto out;
+                }
+                if (!IA_ISREG (sh->type)) {
+                        afr_sh_data_finish (frame, this);
+                        goto out;
+                }
+                GF_ASSERT (sh->old_loop_frame);
+                afr_sh_data_lock (frame, this, 0, 0, _gf_true, this->name,
+                                  afr_post_sh_big_lock_success,
+                                  afr_post_sh_big_lock_failure);
+        }
+out:
+        return 0;
+}
 
-	return 0;
+int
+afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
+{
+        afr_sh_erase_pending (frame, this, AFR_DATA_TRANSACTION,
+                              afr_sh_data_erase_pending_cbk,
+                              afr_sh_data_finish);
+        return 0;
 }
 
+int
+afr_sh_data_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata)
+{
+        afr_local_t     *local      = NULL;
+        afr_private_t   *priv       = NULL;
+        afr_self_heal_t *sh         = NULL;
+        int             call_count  = 0;
+        int             child_index = (long) cookie;
+
+        local = frame->local;
+        priv = this->private;
+        sh   = &local->self_heal;
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to fsync on "
+                        "%s - %s", local->loc.path,
+                        priv->children[child_index]->name, strerror (op_errno));
+                LOCK (&frame->lock);
+                {
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                }
+                UNLOCK (&frame->lock);
+                if (sh->old_loop_frame)
+                        sh_loop_finish (sh->old_loop_frame, this);
+                sh->old_loop_frame = NULL;
+        }
+
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC))
+                        afr_sh_data_fail (frame, this);
+                else
+                        afr_sh_data_erase_pending (frame, this);
+        }
+        return 0;
+}
 
+/*
+ * Before erasing xattrs, make sure the data is written to disk
+ */
 int
-afr_sh_data_erase_pending_cbk (call_frame_t *frame, void *cookie,
-			       xlator_t *this, int32_t op_ret,
-			       int32_t op_errno, dict_t *xattr)
+afr_sh_data_fsync (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int             call_count = 0;
+        afr_local_t     *local     = NULL;
+        afr_private_t   *priv      = NULL;
+        afr_self_heal_t *sh        = NULL;
+        int             i          = 0;
+        int             call_count = 0;
+
+        local = frame->local;
+        priv = this->private;
+        sh   = &local->self_heal;
+
+        call_count        = sh->active_sinks;
+        if (call_count == 0) {
+                afr_sh_data_erase_pending (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!sh->success[i] || sh->sources[i])
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_sh_data_fsync_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->fsync,
+                                   sh->healing_fd, 1, NULL);
+        }
+
+        return 0;
+}
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+static struct afr_sh_algorithm *
+sh_algo_from_name (xlator_t *this, char *name)
+{
+        int i = 0;
 
-	LOCK (&frame->lock);
-	{
-	}
-	UNLOCK (&frame->lock);
+        if (name == NULL)
+                goto out;
 
-	call_count = afr_frame_return (frame);
+        while (afr_self_heal_algorithms[i].name) {
+                if (!strcmp (name, afr_self_heal_algorithms[i].name)) {
+                        return &afr_self_heal_algorithms[i];
+                }
 
-	if (call_count == 0)
-		afr_sh_data_finish (frame, this);
+                i++;
+        }
 
-	return 0;
+out:
+        return NULL;
 }
 
 
-int
-afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
+static int
+sh_zero_byte_files_exist (afr_local_t *local, int child_count)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              i = 0;
-	dict_t          **erase_xattr = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        int             i = 0;
+        int             ret = 0;
+        afr_self_heal_t *sh = NULL;
+
+        sh = &local->self_heal;
+        for (i = 0; i < child_count; i++) {
+                if (!local->child_up[i] || sh->child_errno[i])
+                        continue;
+                if (sh->buf[i].ia_size == 0) {
+                        ret = 1;
+                        break;
+                }
+        }
+
+        return ret;
+}
 
 
-	afr_sh_pending_to_delta (sh->pending_matrix, sh->delta_matrix,
-				 sh->success, priv->child_count);
+struct afr_sh_algorithm *
+afr_sh_data_pick_algo (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t *           priv  = NULL;
+        struct afr_sh_algorithm * algo  = NULL;
+        afr_local_t *             local = NULL;
+        afr_self_heal_t *         sh    = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+        sh    = &local->self_heal;
+        algo  = sh_algo_from_name (this, priv->data_self_heal_algorithm);
+
+        if (algo == NULL) {
+                /* option not set, so fall back on heuristics */
+
+                if (sh_zero_byte_files_exist (local, priv->child_count)
+                    || (sh->file_size <= (priv->data_self_heal_window_size *
+                                          this->ctx->page_size))) {
+
+                        /*
+                         * If the file does not exist on one of the subvolumes,
+                         * or a zero-byte file exists (created by entry self-heal)
+                         * the entire content has to be copied anyway, so there
+                         * is no benefit from using the "diff" algorithm.
+                         *
+                         * If the file size is about the same as page size,
+                         * the entire file can be read and written with a few
+                         * (pipelined) STACK_WINDs, which will be faster
+                         * than "diff" which has to read checksums and then
+                         * read and write.
+                         */
+
+                        algo = sh_algo_from_name (this, "full");
+
+                } else {
+                        algo = sh_algo_from_name (this, "diff");
+                }
+        }
+
+        return algo;
+}
 
-	erase_xattr = CALLOC (sizeof (*erase_xattr), priv->child_count);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i]) {
-			call_count++;
+int
+afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        struct afr_sh_algorithm *sh_algo = NULL;
 
-			erase_xattr[i] = get_new_dict();
-			dict_ref (erase_xattr[i]);
-		}
-	}
+        local = frame->local;
+        sh = &local->self_heal;
 
-	afr_sh_delta_to_xattr (sh->delta_matrix, erase_xattr,
-			       priv->child_count, AFR_DATA_PENDING);
+        sh->algo_completion_cbk = afr_sh_data_fsync;
+        sh->algo_abort_cbk      = afr_sh_data_fail;
 
-	local->call_count = call_count;
-	for (i = 0; i < priv->child_count; i++) {
-		if (!erase_xattr[i])
-			continue;
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"erasing pending flags from %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_data_erase_pending_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->xattrop,
-				   &local->loc,
-				   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-		if (!--call_count)
-			break;
-	}
+        sh_algo = afr_sh_data_pick_algo (frame, this);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (erase_xattr[i]) {
-			dict_unref (erase_xattr[i]);
-		}
-	}
-	FREE (erase_xattr);
+        sh->algo = sh_algo;
+        sh_algo->fn (frame, this);
 
-	return 0;
+        return 0;
 }
 
-
 int
 afr_sh_data_trim_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct stat *buf)
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1)
-			gf_log (this->name, GF_LOG_ERROR,
-				"ftruncate of %s on subvolume %s failed (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-		else
-			gf_log (this->name, GF_LOG_DEBUG,
-				"ftruncate of %s on subvolume %s completed",
-				local->loc.path,
-				priv->children[child_index]->name);
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		afr_sh_data_erase_pending (frame, this);
-	}
-
-	return 0;
+        int              call_count = 0;
+        int              child_index = 0;
+        afr_private_t    *priv = NULL;
+        afr_local_t      *local  = NULL;
+        afr_self_heal_t  *sh = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+        sh    = &local->self_heal;
+
+        child_index = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "ftruncate of %s on subvolume %s failed (%s)",
+                                local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "ftruncate of %s on subvolume %s completed",
+                                local->loc.path,
+                                priv->children[child_index]->name);
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC))
+                        afr_sh_data_fail (frame, this);
+                else
+                        afr_sh_data_sync_prepare (frame, this);
+        }
+
+        return 0;
 }
 
 
 int
 afr_sh_data_trim_sinks (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int             *sources = NULL;
-	int              call_count = 0;
-	int              i = 0;
+        afr_private_t * priv = NULL;
+        afr_local_t * local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        int             *sources = NULL;
+        int              call_count = 0;
+        int              i = 0;
 
 
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
 
-	sources = sh->sources;
-	call_count = sh->active_sinks;
+        sources = sh->sources;
+        call_count = sh->active_sinks;
 
-	local->call_count = call_count;
+        local->call_count = call_count;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i] || !local->child_up[i])
-			continue;
+        for (i = 0; i < priv->child_count; i++) {
+                if (sources[i] || !local->child_up[i])
+                        continue;
 
-		STACK_WIND_COOKIE (frame, afr_sh_data_trim_cbk,
-				   (void *) (long) i,
-				   priv->children[i], 
-				   priv->children[i]->fops->ftruncate,
-				   sh->healing_fd, sh->file_size); 
+                STACK_WIND_COOKIE (frame, afr_sh_data_trim_cbk,
+                                   (void *) (long) i,
+                                   priv->children[i],
+                                   priv->children[i]->fops->ftruncate,
+                                   sh->healing_fd, sh->file_size,
+                                   NULL);
 
-		if (!--call_count)
-			break;
-	}
+                if (!--call_count)
+                        break;
+        }
 
-	return 0;
+        return 0;
 }
 
-
 int
-afr_sh_data_read_write_iter (call_frame_t *frame, xlator_t *this);
+afr_sh_inode_set_read_ctx (afr_self_heal_t *sh, xlator_t *this)
+{
+        afr_private_t   *priv = NULL;
+        int             ret = 0;
+        int             i = 0;
+
+        priv = this->private;
+        sh->source = afr_sh_select_source (sh->sources, priv->child_count);
+        if (sh->source < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        /* detect changes not visible through pending flags -- JIC */
+        for (i = 0; i < priv->child_count; i++) {
+                if (i == sh->source || sh->child_errno[i])
+                        continue;
+
+                if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[sh->source]))
+                        sh->sources[i] = 0;
+        }
+
+        afr_reset_children (sh->fresh_children, priv->child_count);
+        afr_get_fresh_children (sh->success_children, sh->sources,
+                                sh->fresh_children, priv->child_count);
+        afr_inode_set_read_ctx (this, sh->inode, sh->source,
+                                sh->fresh_children);
+out:
+        return ret;
+}
 
-int
-afr_sh_data_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		       int32_t op_ret, int32_t op_errno, struct stat *buf)
+char*
+afr_get_sizes_str (afr_local_t *local, struct iatt *bufs, xlator_t *this)
 {
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-
-	int child_index = (long) cookie;
-	int call_count = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"wrote %d bytes of data from %s to child %d, offset %"PRId64"", 
-		op_ret, local->loc.path, child_index, sh->offset - op_ret);
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"write to %s failed on subvolume %s (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-			sh->op_failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
+        afr_private_t *priv = NULL;
+        int           i     = 0;
+        char          num[1024] = {0};
+        size_t        len = 0;
+        char          *sizes_str = NULL;
+        size_t        off = 0;
+        char          *fmt_str = "%llu bytes on %s, ";
+        char          *child_down =  " %s,";
+        char          *child_unknown = " %s,";
+        int           down_child_present = 0;
+        int           down_count = 0;
+        int           unknown_count = 0;
+        int           unknown_child_present = 0;
+        char          *down_subvol_1 = " down subvolume is ";
+        char          *unknown_subvol_1 = " unknown subvolume is ";
+        char          *down_subvol_2 = " down subvolumes are ";
+        char          *unknown_subvol_2 = " unknown subvolumes are ";
+
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 1) {
+                        len += snprintf (num, sizeof (num), fmt_str,
+                                         (unsigned long long) bufs[i].ia_size,
+                                         priv->children[i]->name);
+                } else if (local->child_up[i] == 0) {
+                        len += snprintf (num, sizeof (num), child_down,
+                                         priv->children[i]->name);
+                        if (!down_child_present)
+                                down_child_present = 1;
+                        down_count ++;
+                } else if (local->child_up[i] == -1) {
+                        len += snprintf (num, sizeof (num), child_unknown,
+                                         priv->children[i]->name);
+                        if (!unknown_child_present)
+                                unknown_child_present = 1;
+                        unknown_count++;
+                }
+
+        }
+
+        if (down_child_present) {
+                if (down_count > 1)
+                        len += snprintf (num, sizeof (num), "%s",
+                                         down_subvol_2);
+                else
+                        len += snprintf (num, sizeof (num), "%s",
+                                        down_subvol_1);
+        }
+        if (unknown_child_present) {
+                if (unknown_count > 1)
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_2);
+                else
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_1);
+        }
+
+        len++;//for '\0'
+
+        sizes_str = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+
+        if (!sizes_str)
+                return NULL;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 1) {
+                        off += snprintf (sizes_str + off, len - off, fmt_str,
+                                         (unsigned long long) bufs[i].ia_size,
+                                         priv->children[i]->name);
+                }
+        }
+
+        if (down_child_present) {
+                if (down_count > 1) {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                         down_subvol_2);
+                } else {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                         down_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 0) {
+                        off += snprintf (sizes_str + off, len - off, child_down,
+                                         priv->children[i]->name);
+                }
+        }
+
+        if (unknown_child_present) {
+                if (unknown_count > 1) {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                        unknown_subvol_2);
+                } else {
+                        off += snprintf (sizes_str + off, len - off, "%s",
+                                         unknown_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == -1) {
+                        off += snprintf (sizes_str + off, len - off,
+                                         child_unknown,
+                                         priv->children[i]->name);
+
+                }
+        }
+
+        return sizes_str;
+}
 
-	call_count = afr_frame_return (frame);
+char*
+afr_get_sinks_str (xlator_t *this, afr_local_t *local, afr_self_heal_t *sh)
+{
+        afr_private_t   *priv = NULL;
+        int             i = 0;
+        char            num[1024] = {0};
+        size_t          len = 0;
+        char            *sinks_str = NULL;
+        char            *temp_str = " to sinks ";
+        char            *str_format = " %s,";
+        char            off = 0;
 
-	if (call_count == 0) {
-		afr_sh_data_read_write_iter (frame, this);
-	}
+        priv = this->private;
 
-	return 0;
-}
+        len += snprintf (num, sizeof (num), "%s", temp_str);
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->sources[i] == 0) && (local->child_up[i] == 1)) {
+                        len += snprintf (num, sizeof (num), str_format,
+                                         priv->children[i]->name);
+                }
+        }
 
+        len ++;
 
-int
-afr_sh_data_read_cbk (call_frame_t *frame, void *cookie,
-		      xlator_t *this, int32_t op_ret, int32_t op_errno,
-		      struct iovec *vector, int32_t count, struct stat *buf)
-{
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
+        sinks_str = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
 
-	int child_index = (long) cookie;
-	int i = 0;
-	int call_count = 0;
+        if (!sinks_str)
+                return NULL;
 
-	off_t offset;
+        off += snprintf (sinks_str + off, len - off, "%s", temp_str);
 
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->sources[i] == 0) && (local->child_up[i] == 1)) {
+                        off += snprintf (sinks_str + off, len - off,
+                                         str_format,
+                                         priv->children[i]->name);
+                }
+        }
 
-	call_count = sh->active_sinks;
+        return sinks_str;
 
-	local->call_count = call_count;
+}
 
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"read %d bytes of data from %s on child %d, offset %"PRId64"",
-		op_ret, local->loc.path, child_index, sh->offset);
 
-	if (op_ret <= 0) {
-		afr_sh_data_trim_sinks (frame, this);
-		return 0;
-	}
+void
+afr_set_data_sh_info_str (afr_local_t *local, afr_self_heal_t *sh, xlator_t *this)
+{
+        char            *pending_matrix_str = NULL;
+        char            *sizes_str = NULL;
+        char            *sinks_str = NULL;
+        afr_private_t   *priv = NULL;
 
-	/* what if we read less than block size? */
-	offset = sh->offset;
-	sh->offset += op_ret;
+        priv = this->private;
 
-	frame->root->req_refs = frame->root->rsp_refs;
+        pending_matrix_str = afr_get_pending_matrix_str (sh->pending_matrix,
+                                                         this);
+        if (!pending_matrix_str)
+                pending_matrix_str = "";
 
-	if (sh->file_has_holes) {
-		if (iov_0filled (vector, count) == 0) {
-			/* the iter function depends on the
-			   sh->offset already being updated 
-			   above
-			*/
-			afr_sh_data_read_write_iter (frame, this);
-			goto out;
-		}
-	}
+        sizes_str = afr_get_sizes_str (local, sh->buf, this);
+        if (!sizes_str)
+                sizes_str = "";
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] || !local->child_up[i])
-			continue;
-
-		/* this is a sink, so write to it */
-		STACK_WIND_COOKIE (frame, afr_sh_data_write_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->writev,
-				   sh->healing_fd, vector, count, offset);
-
-		if (!--call_count)
-			break;
-	}
+        sinks_str = afr_get_sinks_str (this, local, sh);
+        if (!sinks_str)
+                sinks_str = "";
 
-out:
-	return 0;
+        gf_asprintf (&sh->data_sh_info, " data self heal from %s %s with "
+                     "%s data %s", priv->children[sh->source]->name, sinks_str,
+                     sizes_str, pending_matrix_str);
+
+        if (pending_matrix_str && strcmp (pending_matrix_str, ""))
+                GF_FREE (pending_matrix_str);
+
+        if (sizes_str && strcmp (sizes_str, ""))
+                GF_FREE (sizes_str);
 }
 
+void
+afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
+{
+        int              source = 0;
+        afr_local_t     *local      = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        source     = sh->source;
+        sh->block_size = this->ctx->page_size;
+        sh->file_size  = sh->buf[source].ia_size;
+
+        if (FILE_HAS_HOLES (&sh->buf[source]))
+                sh->file_has_holes = 1;
+
+        if (sh->background && sh->unwind && !sh->unwound) {
+                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
+                            is_self_heal_failed (sh, AFR_CHECK_SPECIFIC));
+                sh->unwound = _gf_true;
+        }
+
+        afr_sh_mark_source_sinks (frame, this);
+        if (sh->active_sinks == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no active sinks for performing self-heal on file %s",
+                        local->loc.path);
+                afr_sh_data_finish (frame, this);
+                return;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "self-healing file %s from subvolume %s to %d other",
+                local->loc.path, priv->children[sh->source]->name,
+                sh->active_sinks);
+
+        sh->actual_sh_started = _gf_true;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_SYNC_BEGIN);
+        afr_sh_data_trim_sinks (frame, this);
+}
 
 int
-afr_sh_data_read_write (call_frame_t *frame, xlator_t *this)
+afr_sh_data_fxattrop_fstat_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	STACK_WIND_COOKIE (frame, afr_sh_data_read_cbk,
-			   (void *) (long) sh->source,
-			   priv->children[sh->source],
-			   priv->children[sh->source]->fops->readv,
-			   sh->healing_fd, sh->block_size,
-			   sh->offset);
-
-	return 0;
+        afr_local_t     *local      = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              nsources = 0;
+        int              ret = 0;
+        int             *old_sources = NULL;
+        int             tstamp_source = 0;
+        int             i = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Pending matrix for: %s",
+                lkowner_utoa (&frame->root->lk_owner));
+        if (sh->sync_done) {
+                //store sources before sync so that mtime can be set using the
+                //iatt buf from one of them.
+                old_sources = alloca (priv->child_count*sizeof (*old_sources));
+                memcpy (old_sources, sh->sources,
+                        priv->child_count * sizeof (*old_sources));
+        }
+
+        nsources = afr_build_sources (this, sh->xattr, sh->buf, sh->pending_matrix,
+                                      sh->sources, sh->success_children,
+                                      AFR_DATA_TRANSACTION, NULL, _gf_true);
+        if ((nsources == -1)
+            && (priv->favorite_child != -1)
+            && (sh->child_errno[priv->favorite_child] == 0)) {
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Picking favorite child %s as authentic source to "
+                        "resolve conflicting data of %s",
+                        priv->children[priv->favorite_child]->name,
+                        local->loc.path);
+
+                sh->sources[priv->favorite_child] = 1;
+
+                nsources = afr_sh_source_count (sh->sources,
+                                                priv->child_count);
+        }
+
+        if (nsources == -1) {
+                afr_sh_print_split_brain_log (sh->pending_matrix, this,
+                                              local->loc.path);
+                afr_set_split_brain (this, sh->inode, DONT_KNOW, SPB);
+
+                afr_sh_data_fail (frame, this);
+                return 0;
+        }
+
+        afr_set_split_brain (this, sh->inode, DONT_KNOW, NO_SPB);
+
+        ret = afr_sh_inode_set_read_ctx (sh, this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No active sources found.");
+
+                afr_sh_data_fail (frame, this);
+                return 0;
+        }
+
+        if (sh->sync_done) {
+                /* Perform setattr from one of the old_sources if possible
+                 * Because only they have the correct mtime, the new sources
+                 * (i.e. old sinks) have mtime from last writev in sync.
+                 */
+                tstamp_source = sh->source;
+                for (i = 0; i < priv->child_count; i++) {
+                        if (old_sources[i] && sh->sources[i])
+                                tstamp_source = i;
+                }
+                afr_sh_data_setattr (frame, this, &sh->buf[tstamp_source]);
+        } else {
+                afr_set_data_sh_info_str (local, sh, this);
+                if (nsources == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "No self-heal needed for %s",
+                                local->loc.path);
+
+                        afr_sh_data_finish (frame, this);
+                        return 0;
+                }
+
+                if (sh->do_data_self_heal &&
+                    afr_data_self_heal_enabled (priv->data_self_heal))
+                        afr_sh_data_fix (frame, this);
+                else
+                        afr_sh_data_finish (frame, this);
+        }
+        return 0;
 }
 
-
 int
-afr_sh_data_read_write_iter (call_frame_t *frame, xlator_t *this)
+afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
+                                          dict_t **xattr,
+                                          afr_transaction_type txn_type,
+                                          uuid_t gfid)
 {
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
+        afr_private_t            *priv      = NULL;
+        int                      read_child = -1;
+        int32_t                  **pending_matrix = NULL;
+        int32_t                  *sources         = NULL;
+        int32_t                  *success_children   = NULL;
+        struct iatt              *bufs            = NULL;
+        int32_t                  nsources         = 0;
+        int32_t                  prev_read_child  = -1;
+        int32_t                  config_read_child = -1;
+        int32_t                  subvol_status = 0;
+
+        priv = this->private;
+        bufs = local->cont.lookup.bufs;
+        success_children = local->cont.lookup.success_children;
+
+        pending_matrix = local->cont.lookup.pending_matrix;
+        sources = local->cont.lookup.sources;
+        memset (sources, 0, sizeof (*sources) * priv->child_count);
+
+        nsources = afr_build_sources (this, xattr, bufs, pending_matrix,
+                                      sources, success_children, txn_type,
+                                      &subvol_status, _gf_false);
+        if (subvol_status & SPLIT_BRAIN) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s: Possible split-brain",
+                        local->loc.path);
+                switch (txn_type) {
+                case AFR_DATA_TRANSACTION:
+                        local->cont.lookup.possible_spb = _gf_true;
+                        nsources = 1;
+                        sources[success_children[0]] = 1;
+                        break;
+                case AFR_ENTRY_TRANSACTION:
+                        read_child = afr_get_no_xattr_dir_read_child (this,
+                                                             success_children,
+                                                             bufs);
+                        sources[read_child] = 1;
+                        nsources = 1;
+                        break;
+                default:
+                        break;
+                }
+        }
+        if (nsources < 0)
+                goto out;
+
+        prev_read_child = local->read_child_index;
+        config_read_child = priv->read_child;
+        read_child = afr_select_read_child_from_policy (success_children,
+                                                        priv->child_count,
+                                                        prev_read_child,
+                                                        config_read_child,
+                                                        sources,
+                                                        priv->hash_mode, gfid);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "returning read_child: %d",
+                read_child);
+        return read_child;
+}
 
-	if (sh->op_failed) {
-		afr_sh_data_finish (frame, this);
-		goto out;
-	}
+int
+afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
+                       xlator_t *this, int32_t op_ret, int32_t op_errno,
+                       struct iatt *buf, dict_t *xdata)
+{
+        afr_private_t   *priv  = NULL;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        int call_count  = -1;
+        int child_index = (long) cookie;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret != -1) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "fstat of %s on %s succeeded",
+                                local->loc.path,
+                                priv->children[child_index]->name);
+
+                        sh->buf[child_index] = *buf;
+                        sh->success_children[sh->success_count] = child_index;
+                        sh->success_count++;
+                } else {
+                        gf_log (this->name, GF_LOG_ERROR, "%s: fstat failed "
+                                "on %s, reason %s", local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        sh->child_errno[child_index] = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                /* Previous versions of glusterfs might have set
+                 * the pending data xattrs which need to be erased
+                 */
+                if (!afr_sh_data_proceed (sh->success_count)) {
+                        gf_log (this->name, GF_LOG_ERROR, "inspecting metadata "
+                                "succeeded on < %d children, aborting "
+                                "self-heal for %s", AFR_SH_MIN_PARTICIPANTS,
+                                local->loc.path);
+                        afr_sh_data_fail (frame, this);
+                        goto out;
+                }
+                afr_sh_data_fxattrop_fstat_done (frame, this);
+        }
+out:
+        return 0;
+}
 
-	if (sh->offset >= sh->file_size) {
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"closing fd's of %s",
-			local->loc.path);
-		afr_sh_data_trim_sinks (frame, this);
 
-		goto out;
-	}
+int
+afr_sh_data_fstat (call_frame_t *frame, xlator_t *this)
+{
+        afr_self_heal_t *sh    = NULL;
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv  = NULL;
+        int             call_count = 0;
+        int             i = 0;
+        int             child = 0;
+        int32_t         *fstat_children = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+        sh    = &local->self_heal;
+
+        fstat_children = memdup (sh->success_children,
+                                 sizeof (*fstat_children) * priv->child_count);
+        if (!fstat_children) {
+                afr_sh_data_fail (frame, this);
+                goto out;
+        }
+        call_count = sh->success_count;
+        local->call_count = call_count;
+
+        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
+        afr_reset_children (sh->success_children, priv->child_count);
+        sh->success_count = 0;
+        for (i = 0; i < priv->child_count; i++) {
+                child = fstat_children[i];
+                if (child == -1)
+                        break;
+                STACK_WIND_COOKIE (frame, afr_sh_data_fstat_cbk,
+                                   (void *) (long) child,
+                                   priv->children[child],
+                                   priv->children[child]->fops->fstat,
+                                   sh->healing_fd, NULL);
+                --call_count;
+        }
+        GF_ASSERT (!call_count);
+out:
+        GF_FREE (fstat_children);
+        return 0;
+}
 
-	afr_sh_data_read_write (frame, this);
+void
+afr_sh_common_fxattrop_resp_handler (call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int32_t op_ret,
+                                     int32_t op_errno, dict_t *xattr)
+{
+        afr_private_t   *priv  = NULL;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        int child_index = (long) cookie;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret != -1) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "fxattrop of %s on %s succeeded",
+                                local->loc.path,
+                                priv->children[child_index]->name);
+
+                        sh->xattr[child_index] = dict_ref (xattr);
+                        sh->success_children[sh->success_count] = child_index;
+                        sh->success_count++;
+                } else {
+                        gf_log (this->name, GF_LOG_ERROR, "fxattrop of %s "
+                                "failed on %s, reason %s", local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        sh->child_errno[child_index] = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+}
 
+int
+afr_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
+                          xlator_t *this, int32_t op_ret, int32_t op_errno,
+                          dict_t *xattr, dict_t *xdata)
+{
+        int             call_count  = -1;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh     = NULL;
+
+        local = frame->local;
+        sh    = &local->self_heal;
+
+        afr_sh_common_fxattrop_resp_handler (frame, cookie, this, op_ret,
+                                             op_errno, xattr);
+
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+                if (!afr_sh_data_proceed (sh->success_count)) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s, inspecting "
+                                "change log succeeded on < %d children",
+                                local->loc.path, AFR_SH_MIN_PARTICIPANTS);
+                        afr_sh_data_fail (frame, this);
+                        goto out;
+                }
+                afr_sh_data_fstat (frame, this);
+        }
 out:
-	return 0;
+        return 0;
 }
 
 
 int
-afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, fd_t *fd)
+afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	/* TODO: some of the open's might fail.
-	   In that case, modify cleanup fn to send flush on those 
-	   fd's which are already open */
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"open of %s failed on child %s (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-			sh->op_failed = 1;
-		}
+        afr_self_heal_t *sh    = NULL;
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv  = NULL;
+        dict_t          **xattr_req;
+        int32_t         *zero_pending = NULL;
+        int call_count = 0;
+        int i = 0;
+        int ret = 0;
+	int j;
+
+        priv  = this->private;
+        local = frame->local;
+        sh    = &local->self_heal;
+
+        call_count = afr_up_children_count (local->child_up,
+                                            priv->child_count);
+
+        local->call_count = call_count;
+
+	xattr_req = GF_CALLOC(priv->child_count, sizeof(struct dict_t *),
+			      gf_afr_mt_dict_t);
+	if (!xattr_req)
+		goto out;
 
+	for (i = 0; i < priv->child_count; i++) {
+		xattr_req[i] = dict_new();
+		if (!xattr_req[i]) {
+			ret = -1;
+			goto out;
+		}
 	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
 
-	if (call_count == 0) {
-		if (sh->op_failed) {
-			afr_sh_data_finish (frame, this);
-			return 0;
+	for (i = 0; i < priv->child_count; i++) {
+		for (j = 0; j < priv->child_count; j++) {
+			zero_pending = GF_CALLOC (3, sizeof (*zero_pending),
+						  gf_afr_mt_int32_t);
+			if (!zero_pending) {
+				ret = -1;
+				goto out;
+			}
+			ret = dict_set_dynptr (xattr_req[i], priv->pending_key[j],
+					       zero_pending,
+					       3 * sizeof (*zero_pending));
+			if (ret < 0) {
+				gf_log (this->name, GF_LOG_WARNING,
+					"Unable to set dict value");
+				goto out;
+			} else {
+				zero_pending = NULL;
+			}
 		}
-		gf_log (this->name, GF_LOG_DEBUG,
-			"fd for %s opened, commencing sync",
-			local->loc.path);
+	}
 
-		gf_log (this->name, GF_LOG_WARNING,
-			"sourcing file %s from %s to other sinks",
-			local->loc.path, priv->children[sh->source]->name);
+        afr_reset_xattr (sh->xattr, priv->child_count);
+        afr_reset_children (sh->success_children, priv->child_count);
+        memset (sh->child_errno, 0,
+                sizeof (*sh->child_errno) * priv->child_count);
+        sh->success_count = 0;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_sh_data_fxattrop_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fxattrop,
+                                           sh->healing_fd, GF_XATTROP_ADD_ARRAY,
+                                           xattr_req[i], NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
 
-		afr_sh_data_read_write (frame, this);
+out:
+	if (xattr_req) {
+		for (i = 0; i < priv->child_count; i++)
+			if (xattr_req[i])
+				dict_unref(xattr_req[i]);
+		GF_FREE(xattr_req);
 	}
 
-	return 0;
-}
+        if (ret) {
+                GF_FREE (zero_pending);
+                afr_sh_data_fail (frame, this);
+        }
 
+        return 0;
+}
 
 int
-afr_sh_data_open (call_frame_t *frame, xlator_t *this)
+afr_sh_data_big_lock_success (call_frame_t *frame, xlator_t *this)
 {
-	int i = 0;				
-	int call_count = 0;		     
+        afr_local_t   *local = NULL;
+        afr_self_heal_t *sh = NULL;
 
-	int source = -1;
-	int *sources = NULL;
+        local = frame->local;
+        sh = &local->self_heal;
 
-	fd_t *fd = NULL;
+        sh->data_lock_held = _gf_true;
+        afr_sh_data_fxattrop (frame, this);
+        return 0;
+}
 
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t *sh = NULL;
+int
+afr_sh_dom_lock_success (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_self_heal_t *sh = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
 
-	call_count = sh->active_sinks + 1;
-	local->call_count = call_count;
+        sh->sh_dom_lock_held = _gf_true;
+        afr_sh_data_lock (frame, this, 0, 0, _gf_true, this->name,
+                          afr_sh_data_big_lock_success,
+                          afr_sh_data_fail);
+        return 0;
+}
 
-	fd = fd_create (local->loc.inode, frame->root->pid);
-	sh->healing_fd = fd;
+int
+afr_sh_data_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
 
-	source  = local->self_heal.source;
-	sources = local->self_heal.sources;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        sh       = &local->self_heal;
 
-	sh->block_size = 65536;
-	sh->file_size  = sh->buf[source].st_size;
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Blocking data inodelks "
+                        "failed for %s. by %s",
+                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
 
-	if (FILE_HAS_HOLES (&sh->buf[source]))
-		sh->file_has_holes = 1;
+                sh->data_lock_failure_handler (frame, this);
+        } else {
 
-	/* open source */
-	STACK_WIND_COOKIE (frame, afr_sh_data_open_cbk,
-			   (void *) (long) source,
-			   priv->children[source],
-			   priv->children[source]->fops->open,
-			   &local->loc, O_RDONLY|O_LARGEFILE, fd);
-	call_count--;
+                gf_log (this->name, GF_LOG_DEBUG, "Blocking data inodelks "
+                        "done for %s by %s. Proceding to self-heal",
+                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
 
-	/* open sinks */
-	for (i = 0; i < priv->child_count; i++) {
-		if(sources[i] || !local->child_up[i])
-			continue;
-
-		STACK_WIND_COOKIE (frame, afr_sh_data_open_cbk,
-				   (void *) (long) i,
-				   priv->children[i], 
-				   priv->children[i]->fops->open,
-				   &local->loc, 
-				   O_WRONLY|O_LARGEFILE, fd); 
-
-		if (!--call_count)
-			break;
-	}
+                sh->data_lock_success_handler (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
-
 int
-afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
+afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              active_sinks = 0;
-	int              source = 0;
-	int              i = 0;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        sh       = &local->self_heal;
 
-	source = sh->source;
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
+                        "failed for %s. by %s",
+                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] == 0 && local->child_up[i] == 1) {
-			active_sinks++;
-			sh->success[i] = 1;
+		if (!sh->data_lock_block) {
+			sh->data_lock_failure_handler(frame, this);
+		} else {
+			int_lock->lock_cbk =
+				afr_sh_data_post_blocking_inodelk_cbk;
+			afr_blocking_lock (frame, this);
 		}
-	}
-	sh->success[source] = 1;
-
-	if (active_sinks == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no active sinks for performing self-heal on file %s",
-			local->loc.path);
-		afr_sh_data_finish (frame, this);
-		return 0;
-	}
-	sh->active_sinks = active_sinks;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"syncing data of %s from subvolume %s to %d active sinks",
-		local->loc.path, priv->children[source]->name, active_sinks);
+        } else {
 
-	afr_sh_data_open (frame, this);
+                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
+                        "done for %s by %s. Proceeding to self-heal",
+                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
+                sh->data_lock_success_handler (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
-
 int
-afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
+afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, char *dom,
+                      off_t start, off_t len)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              nsources = 0;
-	int              source = 0;
-	int              i = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	afr_sh_build_pending_matrix (sh->pending_matrix, sh->xattr, 
-				     priv->child_count, AFR_DATA_PENDING);
-
-	afr_sh_print_pending_matrix (sh->pending_matrix, this);
-
-
-	afr_sh_mark_sources (sh->pending_matrix, sh->sources, 
-			     priv->child_count);
+        afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
+        afr_local_t         *local    = NULL;
 
-	afr_sh_supress_empty_children (sh->sources, sh->xattr, sh->buf,
-				       priv->child_count, AFR_DATA_PENDING);
+        local    = frame->local;
+        int_lock = &local->internal_lock;
 
-	afr_sh_supress_errenous_children (sh->sources, sh->child_errno,
-					  priv->child_count);
+        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
+        int_lock->selfheal_lk_type    = AFR_DATA_SELF_HEAL_LK;
 
-	nsources = afr_sh_source_count (sh->sources, priv->child_count);
+        afr_set_lock_number (frame, this);
 
-	if ((nsources == 0)
-	    && (priv->favorite_child != -1)
-	    && (sh->child_errno[priv->favorite_child] == 0)) {
+        int_lock->lock_cbk         = afr_sh_data_post_nonblocking_inodelk_cbk;
 
-		gf_log (this->name, GF_LOG_WARNING,
-			"Picking favorite child %s as authentic source to resolve conflicting data of %s",
-			priv->children[priv->favorite_child]->name,
-			local->loc.path);
+        int_lock->domain = dom;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        inodelk->flock.l_start = start;
+        inodelk->flock.l_len   = len;
+        inodelk->flock.l_type  = F_WRLCK;
 
-		sh->sources[priv->favorite_child] = 1;
+        afr_nonblocking_inodelk (frame, this);
 
-		nsources = afr_sh_source_count (sh->sources,
-						priv->child_count);
-	}
-
-	if (nsources == 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Unable to resolve conflicting data of %s. "
-			"Please resolve manually by deleting the file %s "
-			"from all but the preferred subvolume. "
-			"Please consider 'option favorite-child <>'",
-			local->loc.path, local->loc.path);
-
-		local->govinda_gOvinda = 1;
-
-		afr_sh_data_finish (frame, this);
-		return 0;
-	}
-
-	source = afr_sh_select_source (sh->sources, priv->child_count);
-	sh->source = source;
-
-	/* detect changes not visible through pending flags -- JIC */
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || sh->child_errno[i])
-			continue;
-
-		if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[source]))
-			sh->sources[i] = 0;
-	}
-
-	afr_sh_data_sync_prepare (frame, this);
-
-	return 0;
+        return 0;
 }
 
-
 int
-afr_sh_data_lookup_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this, int32_t op_ret, int32_t op_errno,
-			inode_t *inode, struct stat *buf, dict_t *xattr)
+afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t   *priv  = NULL;
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-			sh->xattr[child_index] = dict_ref (xattr);
-			sh->buf[child_index] = *buf;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		afr_sh_data_fix (frame, this);
-	}
-
-	return 0;
+        afr_local_t     *local     = NULL;
+        afr_self_heal_t *sh        = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        GF_ASSERT (sh->old_loop_frame);
+        sh_loop_finish (sh->old_loop_frame, this);
+        sh->old_loop_frame = NULL;
+        sh->data_lock_held = _gf_true;
+        sh->sync_done = _gf_true;
+        afr_sh_data_fxattrop (frame, this);
+        return 0;
 }
 
-
 int
-afr_sh_data_lookup (call_frame_t *frame, xlator_t *this)
+afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this)
 {
-	afr_self_heal_t *sh    = NULL; 
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv  = NULL;
-	dict_t          *xattr_req = NULL;
-
-	int call_count = 0;
-	int i = 0;
-	int ret = 0;
-
-	priv  = this->private;
-	local = frame->local;
-	sh    = &local->self_heal;
-
-	call_count = local->child_count;
+        afr_local_t     *local     = NULL;
+        afr_self_heal_t *sh        = NULL;
 
-	local->call_count = call_count;
-	
-	xattr_req = dict_new();
-	if (xattr_req)
-		ret = dict_set_uint64 (xattr_req, AFR_DATA_PENDING,
-				       priv->child_count * sizeof(int32_t));
+        local = frame->local;
+        sh = &local->self_heal;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_sh_data_lookup_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->lookup,
-					   &local->loc, xattr_req);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	return 0;
+        GF_ASSERT (sh->old_loop_frame);
+        sh_loop_finish (sh->old_loop_frame, this);
+        sh->old_loop_frame = NULL;
+        afr_sh_set_timestamps (frame, this);
+        return 0;
 }
 
 
 int
-afr_sh_data_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno)
+afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
+                  off_t start, off_t len, gf_boolean_t block,
+                  char *dom, afr_lock_cbk_t success_handler,
+                  afr_lock_cbk_t failure_handler)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	int              call_count = 0;
-	int              child_index = (long) cookie;
-
-	/* TODO: what if lock fails? */
-	
-	local = frame->local;
-	sh = &local->self_heal;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			sh->op_failed = 1;
-
-			gf_log (this->name,
-				(op_errno == EAGAIN ? GF_LOG_DEBUG : GF_LOG_ERROR),
-				"locking of %s on child %d failed: %s",
-				local->loc.path, child_index,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"inode of %s on child %d locked",
-				local->loc.path, child_index);
-		}
-	}
-	UNLOCK (&frame->lock);
+        afr_local_t *   local = NULL;
+        afr_self_heal_t * sh  = NULL;
 
-	call_count = afr_frame_return (frame);
+        local = frame->local;
+        sh    = &local->self_heal;
 
-	if (call_count == 0) {
-		if (sh->op_failed) {
-			afr_sh_data_finish (frame, this);
-			return 0;
-		}
-
-		afr_sh_data_lookup (frame, this);
-	}
-
-	return 0;
+        sh->data_lock_success_handler = success_handler;
+        sh->data_lock_failure_handler = failure_handler;
+	sh->data_lock_block = block;
+        return afr_sh_data_lock_rec (frame, this, dom, start, len);
 }
 
-
 int
-afr_sh_data_lock (call_frame_t *frame, xlator_t *this)
+afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	struct flock flock;			
-	int i = 0;				
-	int call_count = 0;		     
-
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t * sh  = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	call_count = local->child_count;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              call_count = 0;
+        int              child_index = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        child_index = (long) cookie;
+
+        /* TODO: some of the open's might fail.
+           In that case, modify cleanup fn to send flush on those
+           fd's which are already open */
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "open of %s failed on child %s (%s)",
+                                local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "open of %s succeeded on child %s",
+                                local->loc.path,
+                                priv->children[child_index]->name);
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                        afr_sh_data_fail (frame, this);
+                        return 0;
+                }
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "fd for %s opened, commencing sync",
+                        local->loc.path);
+
+                afr_sh_data_lock (frame, this, 0, 0, _gf_true, priv->sh_domain,
+                                  afr_sh_dom_lock_success, afr_sh_data_fail);
+        }
+
+        return 0;
+}
 
-	local->call_count = call_count;		
 
-	flock.l_start = 0;
-	flock.l_len   = 0;
-	flock.l_type  = F_WRLCK;			
+int
+afr_sh_data_open (call_frame_t *frame, xlator_t *this)
+{
+        int i = 0;
+        int call_count = 0;
+        fd_t *fd = NULL;
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        call_count = afr_up_children_count (local->child_up, priv->child_count);
+        local->call_count = call_count;
+
+        fd = fd_create (local->loc.inode, frame->root->pid);
+        sh->healing_fd = fd;
+
+        /* open sinks */
+        for (i = 0; i < priv->child_count; i++) {
+                if(!local->child_up[i])
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_sh_data_open_cbk,
+                                   (void *) (long) i,
+                                   priv->children[i],
+                                   priv->children[i]->fops->open,
+                                   &local->loc,
+                                   O_RDWR|O_LARGEFILE, fd, NULL);
+
+                if (!--call_count)
+                        break;
+        }
+
+        return 0;
+}
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"locking %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame, afr_sh_data_lock_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->inodelk,
-					   &local->loc, F_SETLK, &flock); 
-			if (!--call_count)
-				break;
-		}
-	}
+void
+afr_sh_non_reg_fix (call_frame_t *frame, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno)
+{
+        afr_private_t   *priv = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_local_t     *local = NULL;
+        int             i = 0;
+
+        if (op_ret < 0) {
+                afr_sh_data_fail (frame, this);
+                return;
+        }
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        for (i = 0; i < priv->child_count ; i++) {
+                if (1 == local->child_up[i])
+                        sh->success[i] = 1;
+        }
+
+        afr_sh_erase_pending (frame, this, AFR_DATA_TRANSACTION,
+                              afr_sh_data_erase_pending_cbk,
+                              afr_sh_data_finish);
+}
 
-	return 0;
+int
+afr_sh_non_reg_lock_success (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        sh->data_lock_held = _gf_true;
+        afr_sh_common_lookup (frame, this, &local->loc,
+                              afr_sh_non_reg_fix, NULL,
+                              AFR_LOOKUP_FAIL_CONFLICTS |
+                              AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                              NULL);
+        return 0;
 }
 
+gf_boolean_t
+afr_can_start_data_self_heal (afr_self_heal_t *sh, afr_private_t *priv)
+{
+        if (sh->force_confirm_spb)
+                return _gf_true;
+        if (sh->do_data_self_heal &&
+            afr_data_self_heal_enabled (priv->data_self_heal))
+                return _gf_true;
+        return _gf_false;
+}
 
 int
 afr_self_heal_data (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t   *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t *priv = this->private;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-
-	if (local->need_data_self_heal && priv->data_self_heal) {
-		afr_sh_data_lock (frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"not doing data self heal on %s",
-			local->loc.path);
-		afr_sh_data_done (frame, this);
-	}
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh    = NULL;
+        afr_private_t   *priv  = this->private;
+        int             ret    = -1;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_type_in_action = AFR_SELF_HEAL_DATA;
+
+        if (afr_can_start_data_self_heal (sh, priv)) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
+                ret = afr_inodelk_init (&local->internal_lock.inodelk[1],
+                                        priv->sh_domain, priv->child_count);
+                if (ret < 0) {
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                        afr_sh_data_done (frame, this);
+                        return 0;
+                }
+
+                if (IA_ISREG (sh->type)) {
+                        afr_sh_data_open (frame, this);
+                } else {
+                        afr_sh_data_lock (frame, this, 0, 0, _gf_true,
+                                          this->name,
+                                          afr_sh_non_reg_lock_success,
+                                          afr_sh_data_fail);
+                }
+        } else {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "not doing data self heal on %s",
+                        local->loc.path);
+                afr_sh_data_done (frame, this);
+        }
+
+        return 0;
 }
-
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index ec341922e..53491a1d7 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -30,6 +21,7 @@
 #endif
 
 #include "glusterfs.h"
+#include "inode.h"
 #include "afr.h"
 #include "dict.h"
 #include "xlator.h"
@@ -48,1991 +40,2331 @@
 #include "afr-self-heal.h"
 #include "afr-self-heal-common.h"
 
-
+#define AFR_INIT_SH_FRAME_VALS(_frame, _local, _sh, _sh_frame, _sh_local, _sh_sh)\
+        do {\
+                _local = _frame->local;\
+                _sh = &_local->self_heal;\
+                _sh_frame = _sh->sh_frame;\
+                _sh_local = _sh_frame->local;\
+                _sh_sh    = &_sh_local->self_heal;\
+        } while (0);
 
 int
+afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
+                                  int child_index);
+int
 afr_sh_entry_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	/* 
-	   TODO: cleanup sh->* 
-	*/
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"self heal of %s completed",
-		local->loc.path);
+        local = frame->local;
+        sh = &local->self_heal;
 
-	sh->completion_cbk (frame, this);
+        sh->completion_cbk (frame, this);
 
-	return 0;
-}
-
-
-int
-afr_sh_entry_unlck_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	int           call_count = 0;
-	int           child_index = (long) cookie;
-
-	/* TODO: what if lock fails? */
-	
-	local = frame->local;
-	sh = &local->self_heal;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"unlocking inode of %s on child %d failed: %s",
-				local->loc.path, child_index,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"unlocked inode of %s on child %d",
-				local->loc.path, child_index);
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		if (sh->healing_fd)
-			fd_unref (sh->healing_fd);
-		sh->healing_fd = NULL;
-		afr_sh_entry_done (frame, this);
-	}
-
-	return 0;
+        return 0;
 }
 
 
 int
 afr_sh_entry_unlock (call_frame_t *frame, xlator_t *this)
 {
-	int i = 0;				
-	int call_count = 0;		     
-
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t * sh  = NULL;
+        afr_local_t         *local    = NULL;
+        afr_internal_lock_t *int_lock = NULL;
 
+        local    = frame->local;
+        int_lock = &local->internal_lock;
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        int_lock->lock_cbk = afr_sh_entry_done;
+        afr_unlock (frame, this);
 
-	call_count = local->child_count;
-
-	local->call_count = call_count;		
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"unlocking %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame, afr_sh_entry_unlck_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->entrylk,
-					   &local->loc, NULL,
-					   ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
+        return 0;
 }
 
 
 int
 afr_sh_entry_finish (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t   *local = NULL;
+        afr_local_t   *local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"finishing entry selfheal of %s", local->loc.path);
+        gf_log (this->name, GF_LOG_TRACE,
+                "finishing entry selfheal of %s", local->loc.path);
 
-	afr_sh_entry_unlock (frame, this);
+        afr_sh_entry_unlock (frame, this);
 
-	return 0;
+        return 0;
 }
 
 
 int
 afr_sh_entry_erase_pending_cbk (call_frame_t *frame, void *cookie,
-				xlator_t *this, int32_t op_ret,
-				int32_t op_errno, dict_t *xattr)
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int             call_count = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_entry_finish (frame, this);
-
-	return 0;
+        long                 i          = 0;
+        int                  call_count = 0;
+        afr_local_t         *local      = NULL;
+        afr_self_heal_t     *sh         = NULL;
+        afr_local_t         *orig_local = NULL;
+        call_frame_t        *orig_frame = NULL;
+        afr_private_t       *priv       = NULL;
+        int32_t             read_child  = -1;
+
+        local = frame->local;
+        priv  = this->private;
+        sh = &local->self_heal;
+        i = (long)cookie;
+
+
+        afr_children_add_child (sh->fresh_children, i, priv->child_count);
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: failed to erase pending xattrs on %s (%s)",
+                        local->loc.path, priv->children[i]->name,
+                        strerror (op_errno));
+        }
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                if (sh->source == -1) {
+                        //this happens if the forced merge option is set
+                        read_child = sh->fresh_children[0];
+                } else {
+                        read_child = sh->source;
+                }
+                afr_inode_set_read_ctx (this, sh->inode, read_child,
+                                        sh->fresh_children);
+                orig_frame = sh->orig_frame;
+                orig_local = orig_frame->local;
+
+                if (sh->source != -1) {
+                        orig_local->cont.lookup.buf.ia_nlink = sh->buf[sh->source].ia_nlink;
+                }
+
+                afr_sh_entry_finish (frame, this);
+        }
+
+        return 0;
 }
 
 
 int
 afr_sh_entry_erase_pending (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              i = 0;
-	dict_t          **erase_xattr = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-
-	afr_sh_pending_to_delta (sh->pending_matrix, sh->delta_matrix,
-				 sh->success, priv->child_count);
-
-	erase_xattr = CALLOC (sizeof (*erase_xattr), priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i]) {
-			call_count++;
-
-			erase_xattr[i] = get_new_dict();
-			dict_ref (erase_xattr[i]);
-		}
-	}
-
-	afr_sh_delta_to_xattr (sh->delta_matrix, erase_xattr,
-			       priv->child_count, AFR_ENTRY_PENDING);
-
-	local->call_count = call_count;
-	for (i = 0; i < priv->child_count; i++) {
-		if (!erase_xattr[i])
-			continue;
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"erasing pending flags from %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_entry_erase_pending_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->xattrop,
-				   &local->loc,
-				   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-		if (!--call_count)
-			break;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (erase_xattr[i]) {
-			dict_unref (erase_xattr[i]);
-		}
-	}
-	FREE (erase_xattr);
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        if (sh->entries_skipped) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                goto out;
+        }
+        afr_sh_erase_pending (frame, this, AFR_ENTRY_TRANSACTION,
+                              afr_sh_entry_erase_pending_cbk,
+                              afr_sh_entry_finish);
+        return 0;
+out:
+        afr_sh_entry_finish (frame, this);
+        return 0;
 }
 
 
 
 static int
 next_active_source (call_frame_t *frame, xlator_t *this,
-		    int current_active_source)
+                    int current_active_source)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              source = -1;
-	int              next_active_source = -1;
-	int              i = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	source = sh->source;
-
-	if (source != -1) {
-		if (current_active_source != source)
-			next_active_source = source;
-		goto out;
-	}
-
-	/*
-	  the next active sink becomes the source for the
-	  'conservative decision' of merging all entries
-	*/
-
-	for (i = 0; i < priv->child_count; i++) {
-		if ((sh->sources[i] == 0)
-		    && (local->child_up[i] == 1)
-		    && (i > current_active_source)) {
-
-			next_active_source = i;
-			break;
-		}
-	}
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        int              source = -1;
+        int              next_active_source = -1;
+        int              i = 0;
+
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
+
+        source = sh->source;
+
+        if (source != -1) {
+                if (current_active_source != source)
+                        next_active_source = source;
+                goto out;
+        }
+
+        /*
+          the next active sink becomes the source for the
+          'conservative decision' of merging all entries
+        */
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->sources[i] == 0)
+                    && (local->child_up[i] == 1)
+                    && (i > current_active_source)) {
+
+                        next_active_source = i;
+                        break;
+                }
+        }
 out:
-	return next_active_source;
+        return next_active_source;
 }
 
 
 
 static int
 next_active_sink (call_frame_t *frame, xlator_t *this,
-		  int current_active_sink)
+                  int current_active_sink)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              next_active_sink = -1;
-	int              i = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	/*
-	  the next active sink becomes the source for the
-	  'conservative decision' of merging all entries
-	*/
-
-	for (i = 0; i < priv->child_count; i++) {
-		if ((sh->sources[i] == 0)
-		    && (local->child_up[i] == 1)
-		    && (i > current_active_sink)) {
-
-			next_active_sink = i;
-			break;
-		}
-	}
-
-	return next_active_sink;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        int              next_active_sink = -1;
+        int              i = 0;
+
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
+
+        /*
+          the next active sink becomes the source for the
+          'conservative decision' of merging all entries
+        */
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->sources[i] == 0)
+                    && (local->child_up[i] == 1)
+                    && (i > current_active_sink)) {
+
+                        next_active_sink = i;
+                        break;
+                }
+        }
+
+        return next_active_sink;
 }
 
-
 int
-build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
-{
-	int   ret = -1;
-
-	if (!child) {
-		goto out;
-	}
-
-	if (strcmp (parent->path, "/") == 0)
-		asprintf ((char **)&child->path, "/%s", name);
-	else
-		asprintf ((char **)&child->path, "%s/%s", parent->path, name);
-
-	if (!child->path) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	child->name = strrchr (child->path, '/');
-	if (child->name)
-		child->name++;
-
-	child->parent = inode_ref (parent->inode);
-	child->inode = inode_new (parent->inode->table);
-
-	if (!child->inode) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ret = 0;
-out:
-	if (ret == -1)
-		loc_wipe (child);
-
-	return ret;
-}
+afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this);
 
+int
+afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this);
 
 int
 afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this);
 
 int
 afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-			     int active_src);
+                             int active_src);
 
 int
 afr_sh_entry_expunge_entry_done (call_frame_t *frame, xlator_t *this,
-				 int active_src)
+                                 int active_src, int32_t op_ret,
+                                 int32_t op_errno)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              call_count = 0;
+        int              call_count = 0;
 
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
+        call_count = afr_frame_return (frame);
 
-	LOCK (&frame->lock);
-	{
-	}
-	UNLOCK (&frame->lock);
+        if (call_count == 0)
+                afr_sh_entry_expunge_subvol (frame, this, active_src);
 
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_entry_expunge_subvol (frame, this, active_src);
+        return 0;
+}
 
-	return 0;
+int
+afr_sh_entry_expunge_parent_setattr_cbk (call_frame_t *expunge_frame,
+                                         void *cookie, xlator_t *this,
+                                         int32_t op_ret, int32_t op_errno,
+                                         struct iatt *preop, struct iatt *postop,
+                                         dict_t *xdata)
+{
+        afr_private_t   *priv          = NULL;
+        afr_local_t     *expunge_local = NULL;
+        afr_self_heal_t *expunge_sh    = NULL;
+        call_frame_t    *frame         = NULL;
+        int              active_src    = (long) cookie;
+        afr_self_heal_t *sh            = NULL;
+        afr_local_t     *local         = NULL;
+
+        priv          = this->private;
+        expunge_local = expunge_frame->local;
+        expunge_sh    = &expunge_local->self_heal;
+        frame         = expunge_sh->sh_frame;
+        local         = frame->local;
+        sh            = &local->self_heal;
+
+        if (op_ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "setattr on parent directory of %s on subvolume %s failed: %s",
+                        expunge_local->loc.path,
+                        priv->children[active_src]->name, strerror (op_errno));
+        }
+
+        AFR_STACK_DESTROY (expunge_frame);
+        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
+
+        return 0;
 }
 
 
 int
 afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret, int32_t op_errno)
+                                 xlator_t *this,
+                                 int32_t op_ret, int32_t op_errno,
+                                 struct iatt *preparent,
+                                 struct iatt *postparent, dict_t *xdata)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	int              active_src = 0;
-	call_frame_t    *frame = NULL;
-
-
-	priv = this->private;
-	expunge_local = expunge_frame->local;
-	expunge_sh = &expunge_local->self_heal;
-	frame = expunge_sh->sh_frame;
-
-	active_src = (long) cookie;
-
-	if (op_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"removed %s on %s",
-			expunge_local->loc.path,
-			priv->children[active_src]->name);
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"removing %s on %s failed (%s)",
-			expunge_local->loc.path,
-			priv->children[active_src]->name,
-			strerror (op_errno));
-	}
-
-	AFR_STACK_DESTROY (expunge_frame);
-	afr_sh_entry_expunge_entry_done (frame, this, active_src);
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *expunge_local = NULL;
+        afr_self_heal_t *expunge_sh = NULL;
+        int              active_src = 0;
+        int32_t          valid = 0;
+
+        priv = this->private;
+        expunge_local = expunge_frame->local;
+        expunge_sh = &expunge_local->self_heal;
+
+        active_src = (long) cookie;
+
+        if (op_ret == 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "removed %s on %s",
+                        expunge_local->loc.path,
+                        priv->children[active_src]->name);
+        } else {
+                gf_log (this->name, GF_LOG_INFO,
+                        "removing %s on %s failed (%s)",
+                        expunge_local->loc.path,
+                        priv->children[active_src]->name,
+                        strerror (op_errno));
+        }
+
+        valid = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
+
+        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_parent_setattr_cbk,
+                           (void *) (long) active_src,
+                           priv->children[active_src],
+                           priv->children[active_src]->fops->setattr,
+                           &expunge_sh->parent_loc,
+                           &expunge_sh->parentbuf,
+                           valid, NULL);
+
+        return 0;
 }
 
 
 int
-afr_sh_entry_expunge_rmdir (call_frame_t *expunge_frame, xlator_t *this,
-			     int active_src)
+afr_sh_entry_expunge_unlink (call_frame_t *expunge_frame, xlator_t *this,
+                             int active_src)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *expunge_local = NULL;
 
-	priv = this->private;
-	expunge_local = expunge_frame->local;
+        priv          = this->private;
+        expunge_local = expunge_frame->local;
 
-	gf_log (this->name, GF_LOG_WARNING,
-		"removing directory %s on %s",
-		expunge_local->loc.path, priv->children[active_src]->name);
+        gf_log (this->name, GF_LOG_TRACE,
+                "expunging file %s on %s",
+                expunge_local->loc.path, priv->children[active_src]->name);
 
-	STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-			   (void *) (long) active_src,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->rmdir,
-			   &expunge_local->loc);
+        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
+                           (void *) (long) active_src,
+                           priv->children[active_src],
+                           priv->children[active_src]->fops->unlink,
+                           &expunge_local->loc, 0, NULL);
 
-	return 0;
+        return 0;
 }
 
 
+
 int
-afr_sh_entry_expunge_unlink (call_frame_t *expunge_frame, xlator_t *this,
-			     int active_src)
+afr_sh_entry_expunge_rmdir (call_frame_t *expunge_frame, xlator_t *this,
+                            int active_src)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-
-	priv = this->private;
-	expunge_local = expunge_frame->local;
-
-	gf_log (this->name, GF_LOG_WARNING,
-		"unlinking file %s on %s",
-		expunge_local->loc.path, priv->children[active_src]->name);
-	
-	STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-			   (void *) (long) active_src,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->unlink,
-			   &expunge_local->loc);
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *expunge_local = NULL;
+
+        priv          = this->private;
+        expunge_local = expunge_frame->local;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "expunging directory %s on %s",
+                expunge_local->loc.path, priv->children[active_src]->name);
+
+        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
+                           (void *) (long) active_src,
+                           priv->children[active_src],
+                           priv->children[active_src]->fops->rmdir,
+                           &expunge_local->loc, 1, NULL);
+
+        return 0;
 }
 
 
 int
 afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
-			     int active_src, struct stat *buf)
+                             int active_src, struct iatt *buf,
+                             struct iatt *parentbuf)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	int              source = 0;
-	call_frame_t    *frame = NULL;
-	int              type = 0;
-
-	priv = this->private;
-	expunge_local = expunge_frame->local;
-	expunge_sh = &expunge_local->self_heal;
-	frame = expunge_sh->sh_frame;
-	source = expunge_sh->source;
-
-	type = (buf->st_mode & S_IFMT);
-
-	switch (type) {
-	case S_IFSOCK:
-	case S_IFREG:
-	case S_IFBLK:
-	case S_IFCHR:
-	case S_IFIFO:
-	case S_IFLNK:
-		afr_sh_entry_expunge_unlink (expunge_frame, this, active_src);
-
-		break;
-	case S_IFDIR:
-		afr_sh_entry_expunge_rmdir (expunge_frame, this, active_src);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s has unknown file type on %s: 0%o",
-			expunge_local->loc.path,
-			priv->children[source]->name, type);
-		goto out;
-		break;
-	}
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *expunge_local = NULL;
+        afr_self_heal_t *expunge_sh = NULL;
+        call_frame_t    *frame = NULL;
+        int              type = 0;
+        afr_self_heal_t *sh            = NULL;
+        afr_local_t     *local         = NULL;
+        loc_t           *loc           = NULL;
+
+        priv = this->private;
+        expunge_local = expunge_frame->local;
+        expunge_sh = &expunge_local->self_heal;
+        frame = expunge_sh->sh_frame;
+        local         = frame->local;
+        sh            = &local->self_heal;
+        loc           = &expunge_local->loc;
+
+        type = buf->ia_type;
+        if (loc->parent && uuid_is_null (loc->parent->gfid))
+                uuid_copy (loc->pargfid, parentbuf->ia_gfid);
+
+        switch (type) {
+        case IA_IFSOCK:
+        case IA_IFREG:
+        case IA_IFBLK:
+        case IA_IFCHR:
+        case IA_IFIFO:
+        case IA_IFLNK:
+                afr_sh_entry_expunge_unlink (expunge_frame, this, active_src);
+                break;
+        case IA_IFDIR:
+                afr_sh_entry_expunge_rmdir (expunge_frame, this, active_src);
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s has unknown file type on %s: 0%o",
+                        expunge_local->loc.path,
+                        priv->children[active_src]->name, type);
+                goto out;
+                break;
+        }
+
+        return 0;
 out:
-	AFR_STACK_DESTROY (expunge_frame);
-	afr_sh_entry_expunge_entry_done (frame, this, active_src);
+        AFR_STACK_DESTROY (expunge_frame);
+        sh->expunge_done (frame, this, active_src, -1, EINVAL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 afr_sh_entry_expunge_lookup_cbk (call_frame_t *expunge_frame, void *cookie,
-				xlator_t *this,
-				int32_t op_ret,	int32_t op_errno,
-				inode_t *inode, struct stat *buf, dict_t *x)
+                                 xlator_t *this,
+                                 int32_t op_ret, int32_t op_errno,
+                                 inode_t *inode, struct iatt *buf, dict_t *x,
+                                 struct iatt *postparent)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-
-	priv = this->private;
-	expunge_local = expunge_frame->local;
-	expunge_sh = &expunge_local->self_heal;
-	frame = expunge_sh->sh_frame;
-	active_src = (long) cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"lookup of %s on %s failed (%s)",
-			expunge_local->loc.path,
-			priv->children[active_src]->name,
-			strerror (op_errno));
-		goto out;
-	}
-
-	afr_sh_entry_expunge_remove (expunge_frame, this, active_src, buf);
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *expunge_local = NULL;
+        afr_self_heal_t *expunge_sh = NULL;
+        call_frame_t    *frame = NULL;
+        int              active_src = 0;
+        afr_self_heal_t *sh            = NULL;
+        afr_local_t     *local         = NULL;
+
+        priv = this->private;
+        expunge_local = expunge_frame->local;
+        expunge_sh = &expunge_local->self_heal;
+        frame = expunge_sh->sh_frame;
+        active_src = (long) cookie;
+        local         = frame->local;
+        sh            = &local->self_heal;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "lookup of %s on %s failed (%s)",
+                        expunge_local->loc.path,
+                        priv->children[active_src]->name,
+                        strerror (op_errno));
+                goto out;
+        }
+
+        afr_sh_entry_expunge_remove (expunge_frame, this, active_src, buf,
+                                     postparent);
+
+        return 0;
 out:
-	AFR_STACK_DESTROY (expunge_frame);
-	afr_sh_entry_expunge_entry_done (frame, this, active_src);
+        AFR_STACK_DESTROY (expunge_frame);
+        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
 
-	return 0;
+        return 0;
 }
 
 
 int
 afr_sh_entry_expunge_purge (call_frame_t *expunge_frame, xlator_t *this,
-			    int active_src)
+                            int active_src)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-
-	priv = this->private;
-	expunge_local = expunge_frame->local;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"looking up %s on %s",
-		expunge_local->loc.path, priv->children[active_src]->name);
-	
-	STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_lookup_cbk,
-			   (void *) (long) active_src,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->lookup,
-			   &expunge_local->loc, 0);
-
-	return 0;
-}
+        afr_private_t   *priv = NULL;
+        afr_local_t     *expunge_local = NULL;
+
+        priv = this->private;
+        expunge_local = expunge_frame->local;
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "looking up %s on %s",
+                expunge_local->loc.path, priv->children[active_src]->name);
+
+        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_lookup_cbk,
+                           (void *) (long) active_src,
+                           priv->children[active_src],
+                           priv->children[active_src]->fops->lookup,
+                           &expunge_local->loc, NULL);
 
+        return 0;
+}
 
 int
 afr_sh_entry_expunge_entry_cbk (call_frame_t *expunge_frame, void *cookie,
-				xlator_t *this,
-				int32_t op_ret,	int32_t op_errno,
-				inode_t *inode, struct stat *buf, dict_t *x)
+                                xlator_t *this,
+                                int32_t op_ret, int32_t op_errno,
+                                inode_t *inode, struct iatt *buf, dict_t *x,
+                                struct iatt *postparent)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	int              source = 0;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-
-
-	priv = this->private;
-	expunge_local = expunge_frame->local;
-	expunge_sh = &expunge_local->self_heal;
-	frame = expunge_sh->sh_frame;
-	active_src = expunge_sh->active_source;
-	source = (long) cookie;
-
-	if (op_ret == -1 && op_errno == ENOENT) {
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"missing entry %s on %s",
-			expunge_local->loc.path,
-			priv->children[source]->name);
-
-		afr_sh_entry_expunge_purge (expunge_frame, this, active_src);
-
-		return 0;
-	}
-
-	if (op_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"%s exists under %s",
-			expunge_local->loc.path,
-			priv->children[source]->name);
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"looking up %s under %s failed (%s)",
-			expunge_local->loc.path,
-			priv->children[source]->name,
-			strerror (op_errno));
-	}
-
-	AFR_STACK_DESTROY (expunge_frame);
-	afr_sh_entry_expunge_entry_done (frame, this, active_src);
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *expunge_local = NULL;
+        afr_self_heal_t *expunge_sh = NULL;
+        int              source = 0;
+        call_frame_t    *frame = NULL;
+        int              active_src = 0;
+        int              need_expunge = 0;
+        afr_self_heal_t *sh            = NULL;
+        afr_local_t     *local         = NULL;
+
+        priv = this->private;
+        expunge_local = expunge_frame->local;
+        expunge_sh = &expunge_local->self_heal;
+        frame = expunge_sh->sh_frame;
+        active_src = expunge_sh->active_source;
+        source = (long) cookie;
+        local         = frame->local;
+        sh            = &local->self_heal;
+
+        if (op_ret == -1 && op_errno == ENOENT)
+                need_expunge = 1;
+        else if (op_ret == -1)
+                goto out;
+
+        if (!uuid_is_null (expunge_sh->entrybuf.ia_gfid) &&
+            !uuid_is_null (buf->ia_gfid) &&
+            (uuid_compare (expunge_sh->entrybuf.ia_gfid, buf->ia_gfid) != 0)) {
+                char uuidbuf1[64];
+                char uuidbuf2[64];
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "entry %s found on %s with mismatching gfid (%s/%s)",
+                        expunge_local->loc.path,
+                        priv->children[source]->name,
+                        uuid_utoa_r (expunge_sh->entrybuf.ia_gfid, uuidbuf1),
+                        uuid_utoa_r (buf->ia_gfid, uuidbuf2));
+                need_expunge = 1;
+        }
+
+        if (need_expunge) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Entry %s is missing on %s and deleting from "
+                        "replica's other bricks",
+                        expunge_local->loc.path,
+                        priv->children[source]->name);
+
+                if (postparent)
+                        expunge_sh->parentbuf = *postparent;
+
+                afr_sh_entry_expunge_purge (expunge_frame, this, active_src);
+
+                return 0;
+        }
+
+out:
+        if (op_ret == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s exists under %s",
+                        expunge_local->loc.path,
+                        priv->children[source]->name);
+        } else {
+                gf_log (this->name, GF_LOG_INFO,
+                        "looking up %s under %s failed (%s)",
+                        expunge_local->loc.path,
+                        priv->children[source]->name,
+                        strerror (op_errno));
+        }
+
+        AFR_STACK_DESTROY (expunge_frame);
+        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
+
+        return 0;
 }
 
+static gf_boolean_t
+can_skip_entry_self_heal (char *name, loc_t *parent_loc)
+{
+        if (strcmp (name, ".") == 0) {
+                return _gf_true;
+        } else if (strcmp (name, "..") == 0) {
+                return _gf_true;
+        } else if (loc_is_root (parent_loc) &&
+                   (strcmp (name, GF_REPLICATE_TRASH_DIR) == 0)) {
+                return _gf_true;
+        }
+        return _gf_false;
+}
 
 int
 afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
-			    char *name)
+                            gf_dirent_t *entry)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              ret = -1;
-	call_frame_t    *expunge_frame = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	int              active_src = 0;
-	int              source = 0;
-	int              op_errno = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	active_src = sh->active_source;
-	source = sh->source;
-
-	if ((strcmp (name, ".") == 0)
-	    || (strcmp (name, "..") == 0)) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"skipping inspection of %s under %s",
-			name, local->loc.path);
-		goto out;
-	}
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"inspecting existance of %s under %s",
-		name, local->loc.path);
-
-	expunge_frame = copy_frame (frame);
-	if (!expunge_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (expunge_local, afr_local_t, out);
-
-	expunge_frame->local = expunge_local;
-	expunge_sh = &expunge_local->self_heal;
-	expunge_sh->sh_frame = frame;
-	expunge_sh->active_source = active_src;
-
-	ret = build_child_loc (this, &expunge_local->loc, &local->loc, name);
-	if (ret != 0) {
-		goto out;
-	}
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"looking up %s on %s", expunge_local->loc.path,
-		priv->children[source]->name);
-
-	STACK_WIND_COOKIE (expunge_frame,
-			   afr_sh_entry_expunge_entry_cbk,
-			   (void *) (long) source,
-			   priv->children[source],
-			   priv->children[source]->fops->lookup,
-			   &expunge_local->loc, 0);
-
-	ret = 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        int              ret = -1;
+        call_frame_t    *expunge_frame = NULL;
+        afr_local_t     *expunge_local = NULL;
+        afr_self_heal_t *expunge_sh = NULL;
+        int              active_src = 0;
+        int              source = 0;
+        int              op_errno = 0;
+        char            *name = NULL;
+        int             op_ret = -1;
+
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
+
+        active_src = sh->active_source;
+        source = sh->source;
+        sh->expunge_done = afr_sh_entry_expunge_entry_done;
+
+        name = entry->d_name;
+        if (can_skip_entry_self_heal (name, &local->loc)) {
+                op_ret = 0;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "inspecting existence of %s under %s",
+                name, local->loc.path);
+
+        expunge_frame = copy_frame (frame);
+        if (!expunge_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
+
+        expunge_frame->local = expunge_local;
+        expunge_sh = &expunge_local->self_heal;
+        expunge_sh->sh_frame = frame;
+        expunge_sh->active_source = active_src;
+        expunge_sh->entrybuf = entry->d_stat;
+        loc_copy (&expunge_sh->parent_loc, &local->loc);
+
+        ret = afr_build_child_loc (this, &expunge_local->loc, &local->loc,
+                                   name);
+        if (ret != 0) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "looking up %s on %s", expunge_local->loc.path,
+                priv->children[source]->name);
+
+        STACK_WIND_COOKIE (expunge_frame,
+                           afr_sh_entry_expunge_entry_cbk,
+                           (void *) (long) source,
+                           priv->children[source],
+                           priv->children[source]->fops->lookup,
+                           &expunge_local->loc, NULL);
+
+        ret = 0;
 out:
-	if (ret == -1)
-		afr_sh_entry_expunge_entry_done (frame, this, active_src);
+        if (ret == -1)
+                sh->expunge_done (frame, this, active_src, op_ret, op_errno);
 
-	return 0;
+        return 0;
 }
 
 
 int
 afr_sh_entry_expunge_readdir_cbk (call_frame_t *frame, void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret, int32_t op_errno,
-				  gf_dirent_t *entries)
+                                  xlator_t *this,
+                                  int32_t op_ret, int32_t op_errno,
+                                  gf_dirent_t *entries, dict_t *xdata)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	gf_dirent_t     *entry = NULL;
-	off_t            last_offset = 0;
-	int              active_src = 0;
-	int              entry_count = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	active_src = sh->active_source;
-
-	if (op_ret <= 0) {
-		if (op_ret < 0) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"readdir of %s on subvolume %s failed (%s)",
-				local->loc.path,
-				priv->children[active_src]->name,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"readdir of %s on subvolume %s complete",
-				local->loc.path,
-				priv->children[active_src]->name);
-		}
-
-		afr_sh_entry_expunge_all (frame, this);
-		return 0;
-	}
-
-	list_for_each_entry (entry, &entries->list, list) {
-		last_offset = entry->d_off;
-		entry_count++;
-	}
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"readdir'ed %d entries from %s",
-		entry_count, priv->children[active_src]->name);
-
-	sh->offset = last_offset;
-	local->call_count = entry_count;
-
-	list_for_each_entry (entry, &entries->list, list) {
-		afr_sh_entry_expunge_entry (frame, this, entry->d_name);
-	}
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        gf_dirent_t     *entry = NULL;
+        off_t            last_offset = 0;
+        int              active_src = 0;
+        int              entry_count = 0;
+
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
+
+        active_src = sh->active_source;
+
+        if (op_ret <= 0) {
+                if (op_ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "readdir of %s on subvolume %s failed (%s)",
+                                local->loc.path,
+                                priv->children[active_src]->name,
+                                strerror (op_errno));
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "readdir of %s on subvolume %s complete",
+                                local->loc.path,
+                                priv->children[active_src]->name);
+                }
+
+                afr_sh_entry_expunge_all (frame, this);
+                return 0;
+        }
+
+        list_for_each_entry (entry, &entries->list, list) {
+                last_offset = entry->d_off;
+                entry_count++;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "readdir'ed %d entries from %s",
+                entry_count, priv->children[active_src]->name);
+
+        sh->offset = last_offset;
+        local->call_count = entry_count;
+
+        list_for_each_entry (entry, &entries->list, list) {
+                afr_sh_entry_expunge_entry (frame, this, entry);
+        }
+
+        return 0;
 }
 
 int
 afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-			     int active_src)
+                             int active_src)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
 
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
 
-	STACK_WIND (frame, afr_sh_entry_expunge_readdir_cbk,
-		    priv->children[active_src],
-		    priv->children[active_src]->fops->readdir,
-		    sh->healing_fd, sh->block_size, sh->offset);
+        STACK_WIND (frame, afr_sh_entry_expunge_readdir_cbk,
+                    priv->children[active_src],
+                    priv->children[active_src]->fops->readdirp,
+                    sh->healing_fd, sh->block_size, sh->offset, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              active_src = -1;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        int              active_src = -1;
 
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
 
-	sh->offset = 0;
+        sh->offset = 0;
 
-	if (sh->source == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no active sources for %s to expunge entries",
-			local->loc.path);
-		goto out;
-	}
+        if (sh->source == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no active sources for %s to expunge entries",
+                        local->loc.path);
+                goto out;
+        }
 
-	active_src = next_active_sink (frame, this, sh->active_source);
-	sh->active_source = active_src;
+        active_src = next_active_sink (frame, this, sh->active_source);
+        sh->active_source = active_src;
 
-	if (sh->op_failed) {
-		goto out;
-	}
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                goto out;
+        }
 
-	if (active_src == -1) {
-		/* completed creating missing files on all subvolumes */
-		goto out;
-	}
+        if (active_src == -1) {
+                /* completed creating missing files on all subvolumes */
+                goto out;
+        }
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"expunging entries of %s on %s to other sinks",
-		local->loc.path, priv->children[active_src]->name);
+        gf_log (this->name, GF_LOG_TRACE,
+                "expunging entries of %s on %s to other sinks",
+                local->loc.path, priv->children[active_src]->name);
 
-	afr_sh_entry_expunge_subvol (frame, this, active_src);
+        afr_sh_entry_expunge_subvol (frame, this, active_src);
 
-	return 0;
+        return 0;
 out:
-	afr_sh_entry_erase_pending (frame, this);
-	return 0;
+        afr_sh_entry_impunge_all (frame, this);
+        return 0;
 
 }
 
 
 int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this,
-			     int active_src);
-
-int
 afr_sh_entry_impunge_entry_done (call_frame_t *frame, xlator_t *this,
-				 int active_src)
+                                 int32_t op_ret, int32_t op_errno)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              call_count = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	LOCK (&frame->lock);
-	{
-	}
-	UNLOCK (&frame->lock);
+        int              call_count = 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        if (op_ret < 0)
+                sh->entries_skipped = _gf_true;
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                afr_sh_entry_impunge_subvol (frame, this);
+
+        return 0;
+}
 
-	call_count = afr_frame_return (frame);
+void
+afr_sh_entry_call_impunge_done (call_frame_t *impunge_frame, xlator_t *this,
+                                int32_t op_ret, int32_t op_errno)
+{
+        afr_local_t     *impunge_local = NULL;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_self_heal_t *impunge_sh = NULL;
+        call_frame_t    *frame = NULL;
 
-	if (call_count == 0)
-		afr_sh_entry_impunge_subvol (frame, this, active_src);
+        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
+                                frame, local, sh);
 
-	return 0;
+        AFR_STACK_DESTROY (impunge_frame);
+        sh->impunge_done (frame, this, op_ret, op_errno);
 }
 
-
 int
-afr_sh_entry_impunge_utimens_cbk (call_frame_t *impunge_frame, void *cookie,
-				  xlator_t *this, int32_t op_ret,
-				  int32_t op_errno, struct stat *stbuf)
+afr_sh_entry_impunge_setattr_cbk (call_frame_t *impunge_frame, void *cookie,
+                                  xlator_t *this,
+                                  int32_t op_ret, int32_t op_errno,
+                                  struct iatt *preop, struct iatt *postop,
+                                  dict_t *xdata)
 {
-	int              call_count = 0;
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-	int              child_index = 0;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-	child_index = (long) cookie;
-
-	if (op_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"utimes set for %s on %s",
-			impunge_local->loc.path,
-			priv->children[child_index]->name);
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"setting utimes of %s on %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-	}
-
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
-
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-	}
-
-	return 0;
+        int              call_count = 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        int              child_index = 0;
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+        child_index = (long) cookie;
+
+        if (op_ret == 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "setattr done for %s on %s",
+                        impunge_local->loc.path,
+                        priv->children[child_index]->name);
+        } else {
+                gf_log (this->name, GF_LOG_INFO,
+                        "setattr (%s) on %s failed (%s)",
+                        impunge_local->loc.path,
+                        priv->children[child_index]->name,
+                        strerror (op_errno));
+        }
+
+        call_count = afr_frame_return (impunge_frame);
+        if (call_count == 0) {
+                afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                0, op_errno);
+        }
+
+        return 0;
 }
 
-
 int
-afr_sh_entry_impunge_chown_cbk (call_frame_t *impunge_frame, void *cookie,
-				xlator_t *this, int32_t op_ret,
-				int32_t op_errno, struct stat *stbuf)
+afr_sh_entry_impunge_parent_setattr_cbk (call_frame_t *setattr_frame,
+                                         void *cookie, xlator_t *this,
+                                         int32_t op_ret, int32_t op_errno,
+                                         struct iatt *preop, struct iatt *postop,
+                                         dict_t *xdata)
 {
-	int              call_count = 0;
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-	int              child_index = 0;
-	struct timespec  ts[2];
-
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-	child_index = (long) cookie;
-
-	if (op_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"ownership of %s on %s changed",
-			impunge_local->loc.path,
-			priv->children[child_index]->name);
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"setting ownership of %s on %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-		goto out;
-	}
-
-#ifdef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
-	ts[0] = impunge_local->cont.lookup.buf.st_atim;
-	ts[1] = impunge_local->cont.lookup.buf.st_mtim;
-#elif HAVE_STRUCT_STAT_ST_ATIMESPEC_TV_NSEC
-	ts[0] = impunge_local->cont.lookup.buf.st_atimespec;
-	ts[1] = impunge_local->cont.lookup.buf.st_mtimespec;
-#else
-	ts[0].tv_sec = impunge_local->cont.lookup.buf.st_atime;
-	ts[1].tv_sec = impunge_local->cont.lookup.buf.st_mtime;
-#endif
-	STACK_WIND_COOKIE (impunge_frame,
-			   afr_sh_entry_impunge_utimens_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->utimens,
-			   &impunge_local->loc, ts);
+        int             call_count = 0;
+        afr_local_t     *setattr_local = NULL;
+
+        setattr_local = setattr_frame->local;
+        if (op_ret != 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "setattr on parent directory (%s) failed: %s",
+                        setattr_local->loc.path, strerror (op_errno));
+        }
+
+        call_count = afr_frame_return (setattr_frame);
+        if (call_count == 0)
+                AFR_STACK_DESTROY (setattr_frame);
+        return 0;
+}
 
-	return 0;
+int
+afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
+{
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_local_t     *setattr_local = NULL;
+        afr_self_heal_t *impunge_sh = NULL;
+        call_frame_t    *setattr_frame = NULL;
+        int32_t          valid = 0;
+        int32_t          op_errno = 0;
+        int              child_index = 0;
+        int              call_count = 0;
+        int              i = 0;
+
+        priv          = this->private;
+        impunge_local = impunge_frame->local;
+        impunge_sh    = &impunge_local->self_heal;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "setting ownership of %s on %s to %d/%d",
+                impunge_local->loc.path,
+                priv->children[child_index]->name,
+                impunge_sh->entrybuf.ia_uid,
+                impunge_sh->entrybuf.ia_gid);
+
+        setattr_frame = copy_frame (impunge_frame);
+        if (!setattr_frame) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+        AFR_LOCAL_ALLOC_OR_GOTO (setattr_frame->local, out);
+        setattr_local = setattr_frame->local;
+        call_count = afr_errno_count (NULL, impunge_sh->child_errno,
+                                      priv->child_count, 0);
+        loc_copy (&setattr_local->loc, &impunge_sh->parent_loc);
+        impunge_local->call_count = call_count;
+        setattr_local->call_count = call_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (impunge_sh->child_errno[i])
+                        continue;
+                valid         = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
+                STACK_WIND_COOKIE (setattr_frame,
+                                   afr_sh_entry_impunge_parent_setattr_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->setattr,
+                                   &setattr_local->loc,
+                                   &impunge_sh->parentbuf, valid, NULL);
+
+                valid = GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
+                        GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
+                STACK_WIND_COOKIE (impunge_frame,
+                                   afr_sh_entry_impunge_setattr_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->setattr,
+                                   &impunge_local->loc,
+                                   &impunge_sh->entrybuf, valid, NULL);
+                call_count--;
+        }
+        GF_ASSERT (!call_count);
+        return 0;
+out:
+        if (setattr_frame)
+                AFR_STACK_DESTROY (setattr_frame);
+        afr_sh_entry_call_impunge_done (impunge_frame, this, 0, op_errno);
+        return 0;
+}
 
+int
+afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
+                                  xlator_t *this,
+                                  int32_t op_ret, int32_t op_errno,
+                                  dict_t *xattr, dict_t *xdata)
+{
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        int              child_index = 0;
+
+        priv          = this->private;
+        impunge_local = impunge_frame->local;
+
+        child_index = (long) cookie;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: failed to perform xattrop on %s (%s)",
+                        impunge_local->loc.path,
+                        priv->children[child_index]->name,
+                        strerror (op_errno));
+                goto out;
+        }
+
+        afr_sh_entry_impunge_setattr (impunge_frame, this);
+        return 0;
 out:
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
-
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-	}
-
-	return 0;
+        afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                        -1, op_errno);
+        return 0;
 }
 
+int
+afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
+                                      xlator_t *this)
+{
+        int              active_src       = 0;
+        dict_t          *xattr            = NULL;
+        afr_private_t   *priv             = NULL;
+        afr_local_t     *impunge_local    = NULL;
+        afr_self_heal_t *impunge_sh       = NULL;
+        int32_t         op_errno          = 0;
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+        active_src = impunge_sh->active_source;
+
+        afr_prepare_new_entry_pending_matrix (impunge_local->pending,
+                                              afr_is_errno_unset,
+                                              impunge_sh->child_errno,
+                                              &impunge_sh->entrybuf,
+                                              priv->child_count);
+        xattr = dict_new ();
+        if (!xattr) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        afr_set_pending_dict (priv, xattr, impunge_local->pending, active_src,
+                              LOCAL_LAST);
+
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_xattrop_cbk,
+                           (void *) (long) active_src,
+                           priv->children[active_src],
+                           priv->children[active_src]->fops->xattrop,
+                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr, NULL);
+
+        if (xattr)
+                dict_unref (xattr);
+        return 0;
+out:
+        afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                        -1, op_errno);
+        return 0;
+}
 
 int
 afr_sh_entry_impunge_newfile_cbk (call_frame_t *impunge_frame, void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret, int32_t op_errno,
-				  inode_t *inode, struct stat *stbuf)
+                                  xlator_t *this,
+                                  int32_t op_ret, int32_t op_errno,
+                                  inode_t *inode, struct iatt *stbuf,
+                                  struct iatt *preparent,
+                                  struct iatt *postparent, dict_t *xdata)
 {
-	int              call_count = 0;
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-	int              child_index = 0;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-
-	child_index = (long) cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"creation of %s on %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-		goto out;
-	}
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"setting ownership of %s on %s to %d/%d",
-		impunge_local->loc.path,
-		priv->children[child_index]->name,
-		impunge_local->cont.lookup.buf.st_uid,
-		impunge_local->cont.lookup.buf.st_gid);
-
-	inode->st_mode = stbuf->st_mode;
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_chown_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->chown,
-			   &impunge_local->loc,
-			   impunge_local->cont.lookup.buf.st_uid,
-			   impunge_local->cont.lookup.buf.st_gid);
-	return 0;
-
+        int              call_count       = 0;
+        afr_private_t   *priv             = NULL;
+        afr_local_t     *impunge_local    = NULL;
+        afr_self_heal_t *impunge_sh       = NULL;
+        int              child_index      = 0;
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+
+        child_index = (long) cookie;
+
+        if (op_ret == -1) {
+                impunge_sh->child_errno[child_index] = op_errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "creation of %s on %s failed (%s)",
+                        impunge_local->loc.path,
+                        priv->children[child_index]->name,
+                        strerror (op_errno));
+        } else {
+                impunge_sh->child_errno[child_index] = 0;
+        }
+
+        call_count = afr_frame_return (impunge_frame);
+        if (call_count == 0) {
+                if (!afr_errno_count (NULL, impunge_sh->child_errno,
+                                      priv->child_count, 0)) {
+                        // new_file creation failed every where
+                        afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                        -1, op_errno);
+                        goto out;
+                }
+                afr_sh_entry_impunge_perform_xattrop (impunge_frame, this);
+        }
 out:
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
-
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-	}
-
-	return 0;
+        return 0;
 }
 
-
 int
-afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
-			    int child_index, struct stat *stbuf)
+afr_sh_entry_impunge_hardlink_cbk (call_frame_t *impunge_frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, inode_t *inode,
+                                   struct iatt *buf, struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-
+        int              call_count        = 0;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh  = NULL;
+
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+
+        if (IA_IFLNK == impunge_sh->entrybuf.ia_type) {
+                //For symlinks impunge is attempted un-conditionally
+                //So the file can already exist.
+                if ((op_ret < 0) && (op_errno == EEXIST))
+                        op_ret = 0;
+        }
+
+        call_count = afr_frame_return (impunge_frame);
+        if (call_count == 0)
+                afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                op_ret, op_errno);
+
+        return 0;
+}
 
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
+int
+afr_sh_entry_impunge_hardlink (call_frame_t *impunge_frame, xlator_t *this,
+                               int child_index)
+{
+        afr_private_t   *priv          = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh  = NULL;
+        loc_t           *loc           = NULL;
+        struct iatt     *buf           = NULL;
+        loc_t            oldloc        = {0};
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+        loc = &impunge_local->loc;
+        buf = &impunge_sh->entrybuf;
+
+        oldloc.inode = inode_ref (loc->inode);
+        uuid_copy (oldloc.gfid, buf->ia_gfid);
+        gf_log (this->name, GF_LOG_DEBUG, "linking missing file %s on %s",
+                loc->path, priv->children[child_index]->name);
+
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_hardlink_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->link,
+                           &oldloc, loc, NULL);
+        loc_wipe (&oldloc);
+
+        return 0;
+}
 
-	gf_log (this->name, GF_LOG_WARNING,
-		"creating file %s mode=0%o dev=0x%"GF_PRI_DEV" on %s",
-		impunge_local->loc.path,
-		stbuf->st_mode, stbuf->st_rdev,
-		priv->children[child_index]->name);
+int
+afr_sh_nameless_lookup_cbk (call_frame_t *impunge_frame, void *cookie,
+                            xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, dict_t *xattr,
+                            struct iatt *postparent)
+{
+        if (op_ret < 0) {
+                 afr_sh_entry_impunge_create_file (impunge_frame, this,
+                                                   (long)cookie);
+        } else {
+                afr_sh_entry_impunge_hardlink (impunge_frame, this,
+                                               (long)cookie);
+        }
+        return 0;
+}
 
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->mknod,
-			   &impunge_local->loc,
-			   stbuf->st_mode, stbuf->st_rdev);
+int
+afr_sh_entry_impunge_check_hardlink (call_frame_t *impunge_frame,
+                                     xlator_t *this,
+                                     int child_index, struct iatt *stbuf)
+{
+        afr_private_t   *priv          = NULL;
+        call_frame_t    *frame             = NULL;
+        afr_local_t     *impunge_local     = NULL;
+        afr_local_t     *local             = NULL;
+        afr_self_heal_t *impunge_sh        = NULL;
+        afr_self_heal_t *sh                = NULL;
+        loc_t           *loc           = NULL;
+        dict_t          *xattr_req     = NULL;
+        loc_t            oldloc        = {0};
+        int              ret           = -1;
+
+        priv = this->private;
+        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
+                                frame, local, sh);
+        loc = &impunge_local->loc;
+
+        xattr_req = dict_new ();
+        if (!xattr_req)
+                goto out;
+        oldloc.inode = inode_ref (loc->inode);
+        uuid_copy (oldloc.gfid, stbuf->ia_gfid);
+
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_nameless_lookup_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->lookup,
+                           &oldloc, xattr_req);
+        ret = 0;
+out:
+        if (xattr_req)
+                dict_unref (xattr_req);
+        loc_wipe (&oldloc);
+        if (ret)
+                sh->impunge_done (frame, this, -1, ENOMEM);
+        return 0;
+}
 
-	return 0;
+int
+afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
+                            int child_index, struct iatt *stbuf)
+{
+        afr_private_t *priv          = NULL;
+        afr_local_t   *impunge_local = NULL;
+        dict_t        *dict          = NULL;
+        int            ret           = 0;
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "creating missing file %s on %s",
+                impunge_local->loc.path,
+                priv->children[child_index]->name);
+
+        dict = dict_new ();
+        if (!dict)
+                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
+
+        GF_ASSERT (!uuid_is_null (stbuf->ia_gfid));
+        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
+        if (ret)
+                gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
+                        impunge_local->loc.path);
+
+        /*
+         * Reason for adding GLUSTERFS_INTERNAL_FOP_KEY :
+         *
+         * Problem:
+         * While a brick is down in a replica pair, lets say the user creates
+         * one file(file-A) and a hard link to that file(h-file-A). After the
+         * brick comes back up, entry self-heal is attempted on parent dir of
+         * these two files. As part of readdir in self-heal it reads both the
+         * entries file-A and h-file-A for both of them it does name less lookup
+         * to check if there are any hardlinks already present in the
+         * destination brick. It finds that there are no hard links already
+         * present for files file-A, h-file-A. Self-heal does mknods for both
+         * file-A and h-file-A. This leads to file-A and h-file-A not being
+         * hardlinks anymore.
+         *
+         * Fix: (More like shrinking of race-window, the race itself is still
+         * present in posix-mknod).
+         * If mknod comes with the presence of GLUSTERFS_INTERNAL_FOP_KEY then
+         * posix_mknod checks if there are already any gfid-links and does
+         * link() instead of mknod. There still can be a race where two
+         * posix_mknods same gfid see that
+         * gfid-link file is not present and proceeds with mknods and result in
+         * two different files with same gfid.
+         */
+        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+        if (ret)
+                gf_log (this->name, GF_LOG_INFO, "%s: %s set failed",
+                        impunge_local->loc.path, GLUSTERFS_INTERNAL_FOP_KEY);
+
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->mknod,
+                           &impunge_local->loc,
+                           st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
+                           makedev (ia_major (stbuf->ia_rdev),
+                                    ia_minor (stbuf->ia_rdev)), 0, dict);
+
+        if (dict)
+                dict_unref (dict);
+
+        return 0;
 }
 
 
 
 int
 afr_sh_entry_impunge_mkdir (call_frame_t *impunge_frame, xlator_t *this,
-			    int child_index, struct stat *stbuf)
+                            int child_index, struct iatt *stbuf)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-
-	gf_log (this->name, GF_LOG_WARNING,
-		"creating directory %s mode=0%o on %s",
-		impunge_local->loc.path,
-		stbuf->st_mode,
-		priv->children[child_index]->name);
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->mkdir,
-			   &impunge_local->loc, stbuf->st_mode);
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        dict_t          *dict = NULL;
+
+        int ret = 0;
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+
+        dict = dict_new ();
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Out of memory");
+                return 0;
+        }
+
+        GF_ASSERT (!uuid_is_null (stbuf->ia_gfid));
+        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
+        if (ret)
+                gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
+                        impunge_local->loc.path);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "creating missing directory %s on %s",
+                impunge_local->loc.path,
+                priv->children[child_index]->name);
+
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->mkdir,
+                           &impunge_local->loc,
+                           st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
+                           0, dict);
+
+        if (dict)
+                dict_unref (dict);
+
+        return 0;
 }
 
 
 int
 afr_sh_entry_impunge_symlink (call_frame_t *impunge_frame, xlator_t *this,
-			      int child_index, const char *linkname)
+                              int child_index, const char *linkname)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-
+        afr_private_t   *priv          = NULL;
+        afr_local_t     *impunge_local = NULL;
+        dict_t          *dict          = NULL;
+        struct iatt     *buf           = NULL;
+        int              ret           = 0;
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+
+        buf = &impunge_local->cont.dir_fop.buf;
+
+        dict = dict_new ();
+        if (!dict) {
+                afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                -1, ENOMEM);
+                goto out;
+        }
+
+        GF_ASSERT (!uuid_is_null (buf->ia_gfid));
+        ret = afr_set_dict_gfid (dict, buf->ia_gfid);
+        if (ret)
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: dict set gfid failed",
+                        impunge_local->loc.path);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "creating missing symlink %s -> %s on %s",
+                impunge_local->loc.path, linkname,
+                priv->children[child_index]->name);
+
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->symlink,
+                           linkname, &impunge_local->loc, 0, dict);
+
+        if (dict)
+                dict_unref (dict);
+out:
+        return 0;
+}
 
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
 
-	gf_log (this->name, GF_LOG_WARNING,
-		"creating symlink %s -> %s on %s",
-		impunge_local->loc.path, linkname,
-		priv->children[child_index]->name);
+int
+afr_sh_entry_impunge_symlink_unlink_cbk (call_frame_t *impunge_frame,
+                                         void *cookie, xlator_t *this,
+                                         int32_t op_ret, int32_t op_errno,
+                                         struct iatt *preparent,
+                                         struct iatt *postparent, dict_t *xdata)
+{
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh = NULL;
+        int              child_index = -1;
+        int              call_count = -1;
+
+        priv          = this->private;
+        impunge_local = impunge_frame->local;
+        impunge_sh    = &impunge_local->self_heal;
+
+        child_index = (long) cookie;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "unlink of %s on %s failed (%s)",
+                        impunge_local->loc.path,
+                        priv->children[child_index]->name,
+                        strerror (op_errno));
+                goto out;
+        }
+
+        afr_sh_entry_impunge_symlink (impunge_frame, this, child_index,
+                                      impunge_sh->linkname);
+
+        return 0;
+out:
+        LOCK (&impunge_frame->lock);
+        {
+                call_count = --impunge_local->call_count;
+        }
+        UNLOCK (&impunge_frame->lock);
 
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->symlink,
-			   linkname, &impunge_local->loc);
+        if (call_count == 0)
+                afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                op_ret, op_errno);
 
-	return 0;
+        return 0;
 }
 
 
 int
-afr_sh_entry_impunge_readlink_cbk (call_frame_t *impunge_frame, void *cookie,
-				   xlator_t *this,
-				   int32_t op_ret, int32_t op_errno,
-				   const char *linkname)
+afr_sh_entry_impunge_symlink_unlink (call_frame_t *impunge_frame, xlator_t *this,
+                                     int child_index)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              child_index = -1;
-	call_frame_t    *frame = NULL;
-	int              call_count = -1;
-	int              active_src = -1;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-	active_src = impunge_sh->active_source;
-
-	child_index = (long) cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"readlink of %s on %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[active_src]->name,
-			strerror (op_errno));
-		goto out;
-	}
-
-	afr_sh_entry_impunge_symlink (impunge_frame, this, child_index,
-				      linkname);
-	return 0;
+        afr_private_t   *priv          = NULL;
+        afr_local_t     *impunge_local = NULL;
 
-out:
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
-
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-	}
-
-	return 0;
-}
+        priv          = this->private;
+        impunge_local = impunge_frame->local;
 
+        gf_log (this->name, GF_LOG_DEBUG,
+                "unlinking symlink %s with wrong target on %s",
+                impunge_local->loc.path,
+                priv->children[child_index]->name);
 
-int
-afr_sh_entry_impunge_readlink (call_frame_t *impunge_frame, xlator_t *this,
-			       int child_index, struct stat *stbuf)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              active_src = -1;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	active_src = impunge_sh->active_source;
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_cbk,
-			   (void *) (long) child_index,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->readlink,
-			   &impunge_local->loc, 4096);
-
-	return 0;
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_symlink_unlink_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->unlink,
+                           &impunge_local->loc, 0, NULL);
+
+        return 0;
 }
 
 
 int
-afr_sh_entry_impunge_recreate_lookup_cbk (call_frame_t *impunge_frame,
-					  void *cookie, xlator_t *this,
-					  int32_t op_ret, int32_t op_errno,
-					  inode_t *inode, struct stat *buf,
-					  dict_t *xattr)
+afr_sh_entry_impunge_readlink_sink_cbk (call_frame_t *impunge_frame, void *cookie,
+                                        xlator_t *this,
+                                        int32_t op_ret, int32_t op_errno,
+                                        const char *linkname, struct iatt *sbuf, dict_t *xdata)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              active_src = 0;
-	int              type = 0;
-	int              child_index = 0;
-	call_frame_t    *frame = NULL;
-	int              call_count = 0;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-
-	child_index = (long) cookie;
-
-	active_src = impunge_sh->active_source;
-
-	if (op_ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"looking up %s on %s (for %s) failed (%s)",
-			impunge_local->loc.path,
-			priv->children[active_src]->name,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-		goto out;
-	}
-
-	impunge_local->cont.lookup.buf = *buf;
-	type = (buf->st_mode & S_IFMT);
-
-	switch (type) {
-	case S_IFSOCK:
-	case S_IFREG:
-	case S_IFBLK:
-	case S_IFCHR:
-	case S_IFIFO:
-		afr_sh_entry_impunge_mknod (impunge_frame, this,
-					    child_index, buf);
-		break;
-	case S_IFLNK:
-		afr_sh_entry_impunge_readlink (impunge_frame, this,
-					       child_index, buf);
-		break;
-	case S_IFDIR:
-		afr_sh_entry_impunge_mkdir (impunge_frame, this,
-					    child_index, buf);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s has unknown file type on %s: 0%o",
-			impunge_local->loc.path,
-			priv->children[active_src]->name, type);
-		goto out;
-		break;
-	}
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh = NULL;
+        int              child_index = -1;
+        int              call_count = -1;
+        int              active_src = -1;
+
+        priv          = this->private;
+        impunge_local = impunge_frame->local;
+        impunge_sh    = &impunge_local->self_heal;
+        active_src    = impunge_sh->active_source;
+
+        child_index = (long) cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "readlink of %s on %s failed (%s)",
+                        impunge_local->loc.path,
+                        priv->children[active_src]->name,
+                        strerror (op_errno));
+                goto out;
+        }
+
+        /* symlink doesn't exist on the sink */
+
+        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                afr_sh_entry_impunge_symlink (impunge_frame, this,
+                                              child_index, impunge_sh->linkname);
+                return 0;
+        }
+
+
+        /* symlink exists on the sink, so check if targets match */
+
+        if (strcmp (linkname, impunge_sh->linkname) == 0) {
+                /* targets match, nothing to do */
+
+                goto out;
+        } else {
+                /*
+                 * Hah! Sneaky wolf in sheep's clothing!
+                 */
+                afr_sh_entry_impunge_symlink_unlink (impunge_frame, this,
+                                                     child_index);
+                return 0;
+        }
 
 out:
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
-
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-	}
-
-	return 0;
+        LOCK (&impunge_frame->lock);
+        {
+                call_count = --impunge_local->call_count;
+        }
+        UNLOCK (&impunge_frame->lock);
+
+        if (call_count == 0)
+                afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                op_ret, op_errno);
+
+        return 0;
 }
 
 
 int
-afr_sh_entry_impunge_recreate (call_frame_t *impunge_frame, xlator_t *this,
-			       int child_index)
+afr_sh_entry_impunge_readlink_sink (call_frame_t *impunge_frame, xlator_t *this,
+                                    int child_index)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              active_src = 0;
-
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
 
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
+        priv = this->private;
+        impunge_local = impunge_frame->local;
 
-	active_src = impunge_sh->active_source;
+        gf_log (this->name, GF_LOG_DEBUG,
+                "checking symlink target of %s on %s",
+                impunge_local->loc.path, priv->children[child_index]->name);
 
-	STACK_WIND_COOKIE (impunge_frame,
-			   afr_sh_entry_impunge_recreate_lookup_cbk,
-			   (void *) (long) child_index,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->lookup,
-			   &impunge_local->loc, 0);
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_sink_cbk,
+                           (void *) (long) child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->readlink,
+                           &impunge_local->loc, 4096, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-afr_sh_entry_impunge_entry_cbk (call_frame_t *impunge_frame, void *cookie,
-				xlator_t *this,
-				int32_t op_ret,	int32_t op_errno,
-				inode_t *inode, struct stat *buf, dict_t *x)
+afr_sh_entry_impunge_readlink_cbk (call_frame_t *impunge_frame, void *cookie,
+                                   xlator_t *this,
+                                   int32_t op_ret, int32_t op_errno,
+                                   const char *linkname, struct iatt *sbuf, dict_t *xdata)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-	child_index = (long) cookie;
-	active_src = impunge_sh->active_source;
-
-	if (op_ret == -1 && op_errno == ENOENT) {
-		/* decrease call_count in recreate-callback */
-		gf_log (this->name, GF_LOG_DEBUG,
-			"missing entry %s on %s",
-			impunge_local->loc.path,
-			priv->children[child_index]->name);
-
-		afr_sh_entry_impunge_recreate (impunge_frame, this,
-					       child_index);
-		return 0;
-	}
-
-	if (op_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"%s exists under %s",
-			impunge_local->loc.path,
-			priv->children[child_index]->name);
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"looking up %s under %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-	}
-
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
-
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-	}
-
-	return 0;
-}
-
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh = NULL;
+        int              child_index = -1;
+        int              call_count = -1;
+        int              active_src = -1;
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+        active_src = impunge_sh->active_source;
+
+        child_index = (long) cookie;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "readlink of %s on %s failed (%s)",
+                        impunge_local->loc.path,
+                        priv->children[active_src]->name,
+                        strerror (op_errno));
+                goto out;
+        }
+
+        impunge_sh->linkname = gf_strdup (linkname);
+        afr_sh_entry_impunge_readlink_sink (impunge_frame, this, child_index);
+
+        return 0;
 
-int
-afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
-			    char *name)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              ret = -1;
-	call_frame_t    *impunge_frame = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              active_src = 0;
-	int              i = 0;
-	int              call_count = 0;
-	int              op_errno = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	active_src = sh->active_source;
-
-	if ((strcmp (name, ".") == 0)
-	    || (strcmp (name, "..") == 0)) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"skipping inspection of %s under %s",
-			name, local->loc.path);
-		goto out;
-	}
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"inspecting existance of %s under %s",
-		name, local->loc.path);
-
-	impunge_frame = copy_frame (frame);
-	if (!impunge_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (impunge_local, afr_local_t, out);
-
-	impunge_frame->local = impunge_local;
-	impunge_sh = &impunge_local->self_heal;
-	impunge_sh->sh_frame = frame;
-	impunge_sh->active_source = active_src;
-
-	ret = build_child_loc (this, &impunge_local->loc, &local->loc, name);
-	if (ret != 0) {
-		goto out;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == active_src)
-			continue;
-		if (local->child_up[i] == 0)
-			continue;
-		if (sh->sources[i] == 1)
-			continue;
-		call_count++;
-	}
-
-	impunge_local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == active_src)
-			continue;
-		if (local->child_up[i] == 0)
-			continue;
-		if (sh->sources[i] == 1)
-			continue;
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"looking up %s on %s", impunge_local->loc.path,
-			priv->children[i]->name);
-
-		STACK_WIND_COOKIE (impunge_frame,
-				   afr_sh_entry_impunge_entry_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->lookup,
-				   &impunge_local->loc, 0);
-
-		if (!--call_count)
-			break;
-	}
-
-	ret = 0;
 out:
-	if (ret == -1)
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-	
-	return 0;
+        LOCK (&impunge_frame->lock);
+        {
+                call_count = --impunge_local->call_count;
+        }
+        UNLOCK (&impunge_frame->lock);
+
+        if (call_count == 0)
+                afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                op_ret, op_errno);
+
+        return 0;
 }
 
 
 int
-afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret, int32_t op_errno,
-				  gf_dirent_t *entries)
+afr_sh_entry_impunge_readlink (call_frame_t *impunge_frame, xlator_t *this,
+                               int child_index, struct iatt *stbuf)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	gf_dirent_t     *entry = NULL;
-	off_t            last_offset = 0;
-	int              active_src = 0;
-	int              entry_count = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	active_src = sh->active_source;
-
-	if (op_ret <= 0) {
-		if (op_ret < 0) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"readdir of %s on subvolume %s failed (%s)",
-				local->loc.path,
-				priv->children[active_src]->name,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"readdir of %s on subvolume %s complete",
-				local->loc.path,
-				priv->children[active_src]->name);
-		}
-
-		afr_sh_entry_impunge_all (frame, this);
-		return 0;
-	}
-
-	list_for_each_entry (entry, &entries->list, list) {
-		last_offset = entry->d_off;
-		entry_count++;
-	}
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"readdir'ed %d entries from %s",
-		entry_count, priv->children[active_src]->name);
-
-	sh->offset = last_offset;
-	local->call_count = entry_count;
-
-	list_for_each_entry (entry, &entries->list, list) {
-		afr_sh_entry_impunge_entry (frame, this, entry->d_name);
-	}
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh = NULL;
+        int              active_src = -1;
+
+        priv = this->private;
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+        active_src = impunge_sh->active_source;
+        impunge_local->cont.dir_fop.buf = *stbuf;
+
+        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_cbk,
+                           (void *) (long) child_index,
+                           priv->children[active_src],
+                           priv->children[active_src]->fops->readlink,
+                           &impunge_local->loc, 4096, NULL);
+
+        return 0;
 }
-				  
 
 int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this,
-			     int active_src)
+afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
+                             int child_index)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	STACK_WIND (frame, afr_sh_entry_impunge_readdir_cbk,
-		    priv->children[active_src],
-		    priv->children[active_src]->fops->readdir,
-		    sh->healing_fd, sh->block_size, sh->offset);
-
-	return 0;
+        call_frame_t    *frame             = NULL;
+        afr_local_t     *impunge_local     = NULL;
+        afr_local_t     *local             = NULL;
+        afr_self_heal_t *impunge_sh        = NULL;
+        afr_self_heal_t *sh                = NULL;
+        afr_private_t   *priv = NULL;
+        ia_type_t       type = IA_INVAL;
+        int             active_src = 0;
+        struct iatt     *buf = NULL;
+
+        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
+                                frame, local, sh);
+        active_src = impunge_sh->active_source;
+        afr_update_loc_gfids (&impunge_local->loc, &impunge_sh->entrybuf,
+                              &impunge_sh->parentbuf);
+
+        buf = &impunge_sh->entrybuf;
+        type = buf->ia_type;
+
+        switch (type) {
+        case IA_IFSOCK:
+        case IA_IFREG:
+        case IA_IFBLK:
+        case IA_IFCHR:
+        case IA_IFIFO:
+        case IA_IFLNK:
+                afr_sh_entry_impunge_check_hardlink (impunge_frame, this,
+                                                     child_index, buf);
+                break;
+        case IA_IFDIR:
+                afr_sh_entry_impunge_mkdir (impunge_frame, this,
+                                            child_index, buf);
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s has unknown file type on %s: 0%o",
+                        impunge_local->loc.path,
+                        priv->children[active_src]->name, type);
+                sh->impunge_done (frame, this, -1, EINVAL);
+                break;
+        }
+
+        return 0;
 }
 
-
 int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
+afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
+                                  int child_index)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              active_src = -1;
+        call_frame_t    *frame             = NULL;
+        afr_local_t     *impunge_local     = NULL;
+        afr_local_t     *local             = NULL;
+        afr_self_heal_t *impunge_sh        = NULL;
+        afr_self_heal_t *sh                = NULL;
+        afr_private_t   *priv = NULL;
+        ia_type_t       type = IA_INVAL;
+        int             active_src = 0;
+        struct iatt     *buf = NULL;
+
+        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
+                                frame, local, sh);
+        active_src = impunge_sh->active_source;
+        buf = &impunge_sh->entrybuf;
+        type = buf->ia_type;
+
+        switch (type) {
+        case IA_IFSOCK:
+        case IA_IFREG:
+        case IA_IFBLK:
+        case IA_IFCHR:
+        case IA_IFIFO:
+                afr_sh_entry_impunge_mknod (impunge_frame, this,
+                                            child_index, buf);
+                break;
+        case IA_IFLNK:
+                afr_sh_entry_impunge_readlink (impunge_frame, this,
+                                               child_index, buf);
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s has unknown file type on %s: 0%o",
+                        impunge_local->loc.path,
+                        priv->children[active_src]->name, type);
+                sh->impunge_done (frame, this, -1, EINVAL);
+                break;
+        }
+
+        return 0;
+}
 
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
+gf_boolean_t
+afr_sh_need_recreate (afr_self_heal_t *impunge_sh, unsigned int child,
+                      unsigned int child_count)
+{
+        gf_boolean_t    recreate = _gf_false;
 
-	sh->offset = 0;
+        GF_ASSERT (impunge_sh->child_errno);
 
-	active_src = next_active_source (frame, this, sh->active_source);
-	sh->active_source = active_src;
+        if (child == impunge_sh->active_source)
+                goto out;
 
-	if (sh->op_failed) {
-		afr_sh_entry_finish (frame, this);
-		return 0;
-	}
+        if (IA_IFLNK == impunge_sh->entrybuf.ia_type) {
+                recreate = _gf_true;
+                goto out;
+        }
 
-	if (active_src == -1) {
-		/* completed creating missing files on all subvolumes */
-		afr_sh_entry_expunge_all (frame, this);
-		return 0;
-	}
+        if (impunge_sh->child_errno[child] == ENOENT)
+                recreate = _gf_true;
+out:
+        return recreate;
+}
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"impunging entries of %s on %s to other sinks",
-		local->loc.path, priv->children[active_src]->name);
+unsigned int
+afr_sh_recreate_count (afr_self_heal_t *impunge_sh, int *sources,
+                       unsigned int child_count)
+{
+        int             count = 0;
+        int             i = 0;
 
-	afr_sh_entry_impunge_subvol (frame, this, active_src);
+        for (i = 0; i < child_count; i++) {
+                if (afr_sh_need_recreate (impunge_sh, i, child_count))
+                        count++;
+        }
 
-	return 0;
+        return count;
 }
 
-
 int
-afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			  int32_t op_ret, int32_t op_errno, fd_t *fd)
+afr_sh_entry_call_impunge_recreate (call_frame_t *impunge_frame,
+                                    xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	/* TODO: some of the open's might fail.
-	   In that case, modify cleanup fn to send flush on those 
-	   fd's which are already open */
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"opendir of %s failed on child %s (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-			sh->op_failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		if (sh->op_failed) {
-			afr_sh_entry_finish (frame, this);
-			return 0;
-		}
-		gf_log (this->name, GF_LOG_DEBUG,
-			"fd for %s opened, commencing sync",
-			local->loc.path);
-
-		sh->active_source = -1;
-		afr_sh_entry_impunge_all (frame, this);
-	}
-
-	return 0;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh = NULL;
+        call_frame_t    *frame = NULL;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        unsigned int     recreate_count = 0;
+        int              i = 0;
+        int              active_src = 0;
+
+        priv          = this->private;
+        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
+                                frame, local, sh);
+        active_src = impunge_sh->active_source;
+        impunge_sh->entrybuf = impunge_sh->buf[active_src];
+        impunge_sh->parentbuf = impunge_sh->parentbufs[active_src];
+        recreate_count = afr_sh_recreate_count (impunge_sh, sh->sources,
+                                                priv->child_count);
+        if (!recreate_count) {
+                afr_sh_entry_call_impunge_done (impunge_frame, this, 0, 0);
+                goto out;
+        }
+        impunge_local->call_count = recreate_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!impunge_local->child_up[i]) {
+                        impunge_sh->child_errno[i] = ENOTCONN;
+                        continue;
+                }
+                if (!afr_sh_need_recreate (impunge_sh, i, priv->child_count)) {
+                        impunge_sh->child_errno[i] = EEXIST;
+                        continue;
+                }
+        }
+        for (i = 0; i < priv->child_count; i++) {
+                if (!afr_sh_need_recreate (impunge_sh, i, priv->child_count))
+                        continue;
+                (void)afr_sh_entry_impunge_create (impunge_frame, this, i);
+                recreate_count--;
+        }
+        GF_ASSERT (!recreate_count);
+out:
+        return 0;
 }
 
+void
+afr_sh_entry_common_lookup_done (call_frame_t *impunge_frame, xlator_t *this,
+                                 int32_t op_ret, int32_t op_errno)
+{
+        afr_private_t   *priv = NULL;
+        afr_local_t     *impunge_local = NULL;
+        afr_self_heal_t *impunge_sh = NULL;
+        call_frame_t    *frame = NULL;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        unsigned int     gfid_miss_count = 0;
+        unsigned int     children_up_count = 0;
+        uuid_t           gfid = {0};
+        int              active_src = 0;
+
+        priv          = this->private;
+        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
+                                frame, local, sh);
+        active_src    = impunge_sh->active_source;
+
+        if (op_ret < 0)
+                goto done;
+        if (impunge_sh->child_errno[active_src]) {
+                op_ret = -1;
+                op_errno = impunge_sh->child_errno[active_src];
+                goto done;
+        }
+
+        gfid_miss_count = afr_gfid_missing_count (this->name,
+                                                  impunge_sh->success_children,
+                                                  impunge_sh->buf, priv->child_count,
+                                                  impunge_local->loc.path);
+        children_up_count = afr_up_children_count (impunge_local->child_up,
+                                                   priv->child_count);
+        if ((gfid_miss_count == children_up_count) &&
+            (children_up_count < priv->child_count)) {
+                op_ret = -1;
+                op_errno = ENODATA;
+                gf_log (this->name, GF_LOG_ERROR, "Not all children are up, "
+                        "gfid should not be assigned in this state for %s",
+                        impunge_local->loc.path);
+                goto done;
+        }
+
+        if (gfid_miss_count) {
+                afr_update_gfid_from_iatts (gfid, impunge_sh->buf,
+                                            impunge_sh->success_children,
+                                            priv->child_count);
+                if (uuid_is_null (gfid)) {
+                        sh->entries_skipped = _gf_true;
+                        gf_log (this->name, GF_LOG_INFO, "%s: Skipping entry "
+                                "self-heal because of gfid absence",
+                                impunge_local->loc.path);
+                        goto done;
+                }
+                afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
+                                      afr_sh_entry_common_lookup_done, gfid,
+                                      AFR_LOOKUP_FAIL_CONFLICTS |
+                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                      NULL);
+        } else {
+                afr_sh_entry_call_impunge_recreate (impunge_frame, this);
+        }
+        return;
+done:
+        afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                        op_ret, op_errno);
+        return;
+}
 
 int
-afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
+afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
+                            gf_dirent_t *entry)
 {
-	int i = 0;				
-	int call_count = 0;		     
-
-	int source = -1;
-	int *sources = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        afr_self_heal_t *impunge_sh  = NULL;
+        int              ret = -1;
+        call_frame_t    *impunge_frame = NULL;
+        afr_local_t     *impunge_local = NULL;
+        int              active_src = 0;
+        int              op_errno = 0;
+        int              op_ret = -1;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        active_src = sh->active_source;
+        sh->impunge_done = afr_sh_entry_impunge_entry_done;
+
+        if (can_skip_entry_self_heal (entry->d_name, &local->loc)) {
+                op_ret = 0;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "inspecting existence of %s under %s",
+                entry->d_name, local->loc.path);
+
+        ret = afr_impunge_frame_create (frame, this, active_src,
+                                        &impunge_frame);
+        if (ret) {
+                op_errno = -ret;
+                goto out;
+        }
+
+        impunge_local = impunge_frame->local;
+        impunge_sh = &impunge_local->self_heal;
+        ret = afr_build_child_loc (this, &impunge_local->loc, &local->loc,
+                                   entry->d_name);
+        loc_copy (&impunge_sh->parent_loc, &local->loc);
+        if (ret != 0) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
+                              afr_sh_entry_common_lookup_done, NULL,
+                              AFR_LOOKUP_FAIL_CONFLICTS, NULL);
+
+        op_ret = 0;
+out:
+        if (ret) {
+                if (impunge_frame)
+                        AFR_STACK_DESTROY (impunge_frame);
+                sh->impunge_done (frame, this, op_ret, op_errno);
+        }
 
-	fd_t *fd = NULL;
+        return 0;
+}
 
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t *sh = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+int
+afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
+                                  xlator_t *this,
+                                  int32_t op_ret, int32_t op_errno,
+                                  gf_dirent_t *entries, dict_t *xdata)
+{
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        gf_dirent_t     *entry = NULL;
+        off_t            last_offset = 0;
+        int              active_src = 0;
+        int              entry_count = 0;
+
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
+
+        active_src = sh->active_source;
+
+        if (op_ret <= 0) {
+                if (op_ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "readdir of %s on subvolume %s failed (%s)",
+                                local->loc.path,
+                                priv->children[active_src]->name,
+                                strerror (op_errno));
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "readdir of %s on subvolume %s complete",
+                                local->loc.path,
+                                priv->children[active_src]->name);
+                }
+
+                afr_sh_entry_impunge_all (frame, this);
+                return 0;
+        }
+
+        list_for_each_entry (entry, &entries->list, list) {
+                last_offset = entry->d_off;
+                entry_count++;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "readdir'ed %d entries from %s",
+                entry_count, priv->children[active_src]->name);
+
+        sh->offset = last_offset;
+        local->call_count = entry_count;
+
+        list_for_each_entry (entry, &entries->list, list) {
+                afr_sh_entry_impunge_entry (frame, this, entry);
+        }
+
+        return 0;
+}
 
-	source  = local->self_heal.source;
-	sources = local->self_heal.sources;
 
-	sh->block_size = 131072;
-	sh->offset = 0;
+int
+afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        int32_t         active_src = 0;
+
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
+        active_src = sh->active_source;
+        gf_log (this->name, GF_LOG_DEBUG, "%s: readdir from offset %zd",
+                local->loc.path, sh->offset);
+
+        STACK_WIND (frame, afr_sh_entry_impunge_readdir_cbk,
+                    priv->children[active_src],
+                    priv->children[active_src]->fops->readdirp,
+                    sh->healing_fd, sh->block_size, sh->offset, NULL);
+
+        return 0;
+}
 
-	call_count = sh->active_sinks;
-	if (source != -1)
-		call_count++;
 
-	local->call_count = call_count;
+int
+afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local  = NULL;
+        afr_self_heal_t *sh  = NULL;
+        int              active_src = -1;
 
-	fd = fd_create (local->loc.inode, frame->root->pid);
-	sh->healing_fd = fd;
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
 
-	if (source != -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"opening directory %s on subvolume %s (source)",
-			local->loc.path, priv->children[source]->name);
+        sh->offset = 0;
 
-		/* open source */
-		STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-				   (void *) (long) source,
-				   priv->children[source],
-				   priv->children[source]->fops->opendir,
-				   &local->loc, fd);
-		call_count--;
-	}
+        active_src = next_active_source (frame, this, sh->active_source);
+        sh->active_source = active_src;
 
-	/* open sinks */
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i] || !local->child_up[i])
-			continue;
+        if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                afr_sh_entry_finish (frame, this);
+                return 0;
+        }
 
-		gf_log (this->name, GF_LOG_DEBUG,
-			"opening directory %s on subvolume %s (sink)",
-			local->loc.path, priv->children[i]->name);
+        if (active_src == -1) {
+                /* completed creating missing files on all subvolumes */
+                afr_sh_entry_erase_pending (frame, this);
+                return 0;
+        }
 
-		STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-				   (void *) (long) i,
-				   priv->children[i], 
-				   priv->children[i]->fops->opendir,
-				   &local->loc, fd);
+        gf_log (this->name, GF_LOG_TRACE,
+                "impunging entries of %s on %s to other sinks",
+                local->loc.path, priv->children[active_src]->name);
 
-		if (!--call_count)
-			break;
-	}
+        afr_sh_entry_impunge_subvol (frame, this);
 
-	return 0;
+        return 0;
 }
 
 
 int
-afr_sh_entry_sync_prepare (call_frame_t *frame, xlator_t *this)
+afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              active_sinks = 0;
-	int              source = 0;
-	int              i = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	source = sh->source;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] == 0 && local->child_up[i] == 1) {
-			active_sinks++;
-			sh->success[i] = 1;
-		}
-	}
-	if (source != -1)
-		sh->success[source] = 1;
-
-	if (active_sinks == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no active sinks for self-heal on dir %s",
-			local->loc.path);
-		afr_sh_entry_finish (frame, this);
-		return 0;
-	}
-	if (source == -1 && active_sinks < 2) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"cannot sync with 0 sources and 1 sink on dir %s",
-			local->loc.path);
-		afr_sh_entry_finish (frame, this);
-		return 0;
-	}
-	sh->active_sinks = active_sinks;
-
-	if (source != -1)
-		gf_log (this->name, GF_LOG_DEBUG,
-			"syncing %s from subvolume %s to %d active sinks",
-			local->loc.path, priv->children[source]->name,
-			active_sinks);
-	else
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no active sources for %s found. "
-			"merging all entries as a conservative decision",
-			local->loc.path);
-
-	afr_sh_entry_open (frame, this);
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              call_count = 0;
+        int              child_index = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        child_index = (long) cookie;
+
+        /* TODO: some of the open's might fail.
+           In that case, modify cleanup fn to send flush on those
+           fd's which are already open */
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "opendir of %s failed on child %s (%s)",
+                                local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+                        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                if (is_self_heal_failed (sh, AFR_CHECK_SPECIFIC)) {
+                        afr_sh_entry_finish (frame, this);
+                        return 0;
+                }
+                gf_log (this->name, GF_LOG_TRACE,
+                        "fd for %s opened, commencing sync",
+                        local->loc.path);
+
+                sh->active_source = -1;
+                afr_sh_entry_expunge_all (frame, this);
+        }
+
+        return 0;
 }
 
 
 int
-afr_sh_entry_fix (call_frame_t *frame, xlator_t *this)
+afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              source = 0;
+        int i = 0;
+        int call_count = 0;
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        int source = -1;
+        int *sources = NULL;
 
-	afr_sh_build_pending_matrix (sh->pending_matrix, sh->xattr, 
-				     priv->child_count, AFR_ENTRY_PENDING);
+        fd_t *fd = NULL;
 
-	afr_sh_print_pending_matrix (sh->pending_matrix, this);
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        afr_self_heal_t *sh = NULL;
 
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
 
-	afr_sh_mark_sources (sh->pending_matrix, sh->sources, 
-			     priv->child_count);
+        source  = local->self_heal.source;
+        sources = local->self_heal.sources;
 
-	afr_sh_supress_errenous_children (sh->sources, sh->child_errno,
-					  priv->child_count);
+        sh->block_size = priv->sh_readdir_size;
+        sh->offset = 0;
 
-	source = afr_sh_select_source (sh->sources, priv->child_count);
-	sh->source = source;
+        call_count = sh->active_sinks;
+        if (source != -1)
+                call_count++;
 
-	afr_sh_entry_sync_prepare (frame, this);
+        local->call_count = call_count;
 
-	return 0;
-}
+        fd = fd_create (local->loc.inode, frame->root->pid);
+        sh->healing_fd = fd;
 
+        if (source != -1) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "opening directory %s on subvolume %s (source)",
+                        local->loc.path, priv->children[source]->name);
 
+                /* open source */
+                STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
+                                   (void *) (long) source,
+                                   priv->children[source],
+                                   priv->children[source]->fops->opendir,
+                                   &local->loc, fd, NULL);
+                call_count--;
+        }
 
-int
-afr_sh_entry_lookup_cbk (call_frame_t *frame, void *cookie,
-			 xlator_t *this, int32_t op_ret, int32_t op_errno,
-			 inode_t *inode, struct stat *buf, dict_t *xattr)
-{
-	afr_private_t   *priv  = NULL;
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+        /* open sinks */
+        for (i = 0; i < priv->child_count; i++) {
+                if (sources[i] || !local->child_up[i])
+                        continue;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-			sh->xattr[child_index] = dict_ref (xattr);
-			sh->buf[child_index] = *buf;
-		}
-	}
-	UNLOCK (&frame->lock);
+                gf_log (this->name, GF_LOG_TRACE,
+                        "opening directory %s on subvolume %s (sink)",
+                        local->loc.path, priv->children[i]->name);
 
-	call_count = afr_frame_return (frame);
+                STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
+                                   (void *) (long) i,
+                                   priv->children[i],
+                                   priv->children[i]->fops->opendir,
+                                   &local->loc, fd, NULL);
 
-	if (call_count == 0) {
-		afr_sh_entry_fix (frame, this);
-	}
+                if (!--call_count)
+                        break;
+        }
 
-	return 0;
+        return 0;
 }
 
 
-
 int
-afr_sh_entry_lookup (call_frame_t *frame, xlator_t *this)
+afr_sh_entry_sync_prepare (call_frame_t *frame, xlator_t *this)
 {
-	afr_self_heal_t * sh    = NULL; 
-	afr_local_t    *  local = NULL;
-	afr_private_t  *  priv  = NULL;
-	dict_t         *xattr_req = NULL;
-	int ret = 0;
-	int call_count = 0;
-	int i = 0;
-
-	priv  = this->private;
-	local = frame->local;
-	sh    = &local->self_heal;
-
-	call_count = local->child_count;
-
-	local->call_count = call_count;
-	
-	xattr_req = dict_new();
-	if (xattr_req)
-		ret = dict_set_uint64 (xattr_req, AFR_ENTRY_PENDING,
-				       priv->child_count * sizeof(int32_t));
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame,
-					   afr_sh_entry_lookup_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->lookup,
-					   &local->loc, xattr_req);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              source = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        source = sh->source;
+
+        afr_sh_mark_source_sinks (frame, this);
+        if (source != -1)
+                sh->success[source] = 1;
+
+        if (sh->active_sinks == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "no active sinks for self-heal on dir %s",
+                        local->loc.path);
+                afr_sh_entry_finish (frame, this);
+                return 0;
+        }
+        if (source == -1 && sh->active_sinks < 2) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "cannot sync with 0 sources and 1 sink on dir %s",
+                        local->loc.path);
+                afr_sh_entry_finish (frame, this);
+                return 0;
+        }
+
+        if (source != -1)
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "self-healing directory %s from subvolume %s to "
+                        "%d other",
+                        local->loc.path, priv->children[source]->name,
+                        sh->active_sinks);
+        else
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no active sources for %s found. "
+                        "merging all entries as a conservative decision",
+                        local->loc.path);
+
+        sh->actual_sh_started = _gf_true;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_SYNC_BEGIN);
+        afr_sh_entry_open (frame, this);
+
+        return 0;
 }
 
 
-
-int
-afr_sh_entry_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+void
+afr_sh_entry_fix (call_frame_t *frame, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	int              call_count = 0;
-	int              child_index = (long) cookie;
-
-	/* TODO: what if lock fails? */
-	
-	local = frame->local;
-	sh    = &local->self_heal;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			sh->op_failed = 1;
-
-			gf_log (this->name,
-				(op_errno == EAGAIN ? GF_LOG_DEBUG : GF_LOG_ERROR),
-				"locking inode of %s on child %d failed: %s",
-				local->loc.path, child_index,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"inode of %s on child %d locked",
-				local->loc.path, child_index);
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		if (sh->op_failed == 1) {
-			afr_sh_entry_finish (frame, this);
-			return 0;
-		}
-
-		afr_sh_entry_lookup (frame, this);
-	}
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              source = 0;
+        int              nsources = 0;
+        int32_t          subvol_status = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        if (op_ret < 0) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                afr_sh_set_error (sh, op_errno);
+                afr_sh_entry_finish (frame, this);
+                goto out;
+        }
+
+        if (sh->forced_merge) {
+                sh->source = -1;
+                goto heal;
+        }
+
+        nsources = afr_build_sources (this, sh->xattr, sh->buf,
+                                      sh->pending_matrix, sh->sources,
+                                      sh->success_children,
+                                      AFR_ENTRY_TRANSACTION, &subvol_status,
+                                      _gf_true);
+        if ((subvol_status & ALL_FOOLS) ||
+            (subvol_status & SPLIT_BRAIN)) {
+                gf_log (this->name, GF_LOG_INFO, "%s: Performing conservative "
+                        "merge", local->loc.path);
+                source = -1;
+                memset (sh->sources, 0,
+                        sizeof (*sh->sources) * priv->child_count);
+        } else if (nsources == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "No self-heal needed for %s",
+                        local->loc.path);
+
+                afr_sh_entry_finish (frame, this);
+                return;
+        } else {
+                source = afr_sh_select_source (sh->sources, priv->child_count);
+        }
+
+        sh->source = source;
+
+        afr_reset_children (sh->fresh_children, priv->child_count);
+        afr_get_fresh_children (sh->success_children, sh->sources,
+                                sh->fresh_children, priv->child_count);
+        if (sh->source >= 0)
+                afr_inode_set_read_ctx (this, sh->inode, sh->source,
+                                        sh->fresh_children);
+
+heal:
+        afr_sh_entry_sync_prepare (frame, this);
+out:
+        return;
 }
 
-
 int
-afr_sh_entry_lock (call_frame_t *frame, xlator_t *this)
+afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
 {
-	int i = 0;				
-	int call_count = 0;		     
-
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t * sh  = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	call_count = local->child_count;
-
-	local->call_count = call_count;		
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"locking %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame, afr_sh_entry_lock_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->entrylk,
-					   &local->loc, NULL,
-					   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        sh       = &local->self_heal;
+
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Non Blocking entrylks "
+                        "failed for %s.", local->loc.path);
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                afr_sh_entry_done (frame, this);
+        } else {
+
+                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking entrylks done "
+                        "for %s. Proceeding to FOP", local->loc.path);
+                afr_sh_common_lookup (frame, this, &local->loc,
+                                      afr_sh_entry_fix, NULL,
+                                      AFR_LOOKUP_FAIL_CONFLICTS |
+                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                      NULL);
+        }
+
+        return 0;
 }
 
-
 int
 afr_self_heal_entry (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t   *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	if (local->need_entry_self_heal && priv->entry_self_heal) {
-		afr_sh_entry_lock (frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"proceeding to completion on %s",
-			local->loc.path);
-		afr_sh_entry_done (frame, this);
-	}
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        priv = this->private;
+        local = frame->local;
+        sh = &local->self_heal;
+
+        sh->sh_type_in_action = AFR_SELF_HEAL_ENTRY;
+
+        if (local->self_heal.do_entry_self_heal && priv->entry_self_heal) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
+                afr_sh_entrylk (frame, this, &local->loc, NULL,
+                                afr_sh_post_nonblocking_entry_cbk);
+        } else {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "proceeding to completion on %s",
+                        local->loc.path);
+                afr_sh_entry_done (frame, this);
+        }
+
+        return 0;
 }
-
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index e65a426db..fd5da6cfd 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -52,740 +43,728 @@
 int
 afr_sh_metadata_done (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-//	memset (sh->child_errno, 0, sizeof (int) * priv->child_count);
-	memset (sh->buf, 0, sizeof (struct stat) * priv->child_count);
-	memset (sh->success, 0, sizeof (int) * priv->child_count);
-	
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i])
-			dict_unref (sh->xattr[i]);
-		sh->xattr[i] = NULL;
-	}
-
-	if (local->govinda_gOvinda) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"aborting selfheal of %s",
-			local->loc.path);
-		sh->completion_cbk (frame, this);
-	} else {
-		if (S_ISREG (local->cont.lookup.buf.st_mode)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"proceeding to data check on %s",
-				local->loc.path);
-			afr_self_heal_data (frame, this);
-			return 0;
-		}
-
-		if (S_ISDIR (local->cont.lookup.buf.st_mode)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"proceeding to entry check on %s",
-				local->loc.path);
-			afr_self_heal_entry (frame, this);
-			return 0;
-		}
-		gf_log (this->name, GF_LOG_DEBUG,
-			"completed self heal of %s",
-			local->loc.path);
-
-		sh->completion_cbk (frame, this);
-	}
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+
+        local = frame->local;
+        sh = &local->self_heal;
+
+        afr_sh_reset (frame, this);
+        if (IA_ISDIR (sh->type)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "proceeding to entry check on %s",
+                        local->loc.path);
+                afr_self_heal_entry (frame, this);
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "proceeding to data check on %s",
+                        local->loc.path);
+                afr_self_heal_data (frame, this);
+        }
+
+        return 0;
 }
 
-
 int
-afr_sh_metadata_unlck_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
+afr_sh_inode_unlock (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t      *local = NULL;
-	int               call_count = 0;
-
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
 
-	local = frame->local;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
 
-	LOCK (&frame->lock);
-	{
-	}
-	UNLOCK (&frame->lock);
+        int_lock->lock_cbk = afr_sh_metadata_done;
+        afr_unlock (frame, this);
 
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_metadata_done (frame, this);
-
-	return 0;
+        return 0;
 }
 
-
 int
 afr_sh_metadata_finish (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-	int              call_count = 0;
-	struct flock     flock = {0, };
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	call_count = local->child_count;
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		flock.l_start   = 0;
-		flock.l_len     = 0;
-		flock.l_type    = F_UNLCK;
-
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"unlocking %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND (frame, afr_sh_metadata_unlck_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->inodelk,
-				    &local->loc, F_SETLK, &flock);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
-}
+        afr_sh_inode_unlock (frame, this);
 
+        return 0;
+}
 
 int
-afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
-				   xlator_t *this, int32_t op_ret,
-				   int32_t op_errno, dict_t *xattr)
+afr_sh_metadata_fail (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int             call_count = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-	}
-	UNLOCK (&frame->lock);
+        afr_local_t         *local    = NULL;
+        afr_self_heal_t     *sh       = NULL;
 
-	call_count = afr_frame_return (frame);
+        local    = frame->local;
+        sh       = &local->self_heal;
 
-	if (call_count == 0)
-		afr_sh_metadata_finish (frame, this);
-
-	return 0;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+        afr_sh_metadata_finish (frame, this);
+        return 0;
 }
 
+int
+afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xattr, dict_t *xdata)
+{
+        afr_local_t     *local     = NULL;
+        int             call_count = 0;
+        long            i          = 0;
+        afr_self_heal_t *sh        = NULL;
+        afr_private_t   *priv      = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+        sh = &local->self_heal;
+        i = (long)cookie;
+
+        if ((!IA_ISREG (sh->buf[sh->source].ia_type)) &&
+            (!IA_ISDIR (sh->buf[sh->source].ia_type))) {
+                afr_children_add_child (sh->fresh_children, i,
+                                        priv->child_count);
+        }
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                if ((!IA_ISREG (sh->buf[sh->source].ia_type)) &&
+                    (!IA_ISDIR (sh->buf[sh->source].ia_type))) {
+                        afr_inode_set_read_ctx (this, sh->inode, sh->source,
+                                                sh->fresh_children);
+                }
+                afr_sh_metadata_finish (frame, this);
+        }
+
+        return 0;
+}
 
 int
 afr_sh_metadata_erase_pending (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              i = 0;
-	dict_t          **erase_xattr = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-
-	afr_sh_pending_to_delta (sh->pending_matrix, sh->delta_matrix,
-				 sh->success, priv->child_count);
-
-	erase_xattr = CALLOC (sizeof (*erase_xattr), priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i]) {
-			call_count++;
-
-			erase_xattr[i] = get_new_dict();
-			dict_ref (erase_xattr[i]);
-		}
-	}
-
-	afr_sh_delta_to_xattr (sh->delta_matrix, erase_xattr,
-			       priv->child_count, AFR_METADATA_PENDING);
-
-	local->call_count = call_count;
-
-	if (call_count == 0) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"metadata of %s not healed on any subvolume",
-			local->loc.path);
-
-		afr_sh_metadata_finish (frame, this);
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!erase_xattr[i])
-			continue;
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"erasing pending flags from %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_metadata_erase_pending_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->xattrop,
-				   &local->loc,
-				   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-		if (!--call_count)
-			break;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (erase_xattr[i]) {
-			dict_unref (erase_xattr[i]);
-		}
-	}
-	FREE (erase_xattr);
-
-	return 0;
+         afr_sh_erase_pending (frame, this, AFR_METADATA_TRANSACTION,
+                               afr_sh_metadata_erase_pending_cbk,
+                               afr_sh_metadata_finish);
+         return 0;
 }
 
 
 int
 afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			  int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"setting attributes failed for %s on %s (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-
-			sh->success[child_index] = 0;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_metadata_erase_pending (frame, this);
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              call_count = 0;
+        int              child_index = 0;
+
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        child_index = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "setting attributes failed for %s on %s (%s)",
+                                local->loc.path,
+                                priv->children[child_index]->name,
+                                strerror (op_errno));
+
+                        sh->success[child_index] = 0;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+                if (local->xattr_req) {
+                        dict_unref (local->xattr_req);
+                        local->xattr_req = NULL;
+                }
+                afr_sh_metadata_erase_pending (frame, this);
+        }
+
+        return 0;
 }
 
 
 int
-afr_sh_metadata_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			  int32_t op_ret, int32_t op_errno, struct stat *buf)
+afr_sh_metadata_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-	afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 
-	return 0;
+        return 0;
 }
 
 
 int
 afr_sh_metadata_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 
-	return 0;
+        return 0;
 }
 
-
 int
-afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
+afr_sh_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              source = 0;
-	int              active_sinks = 0;
-	int              call_count = 0;
-	int              i = 0;
-	struct timespec  ts[2];
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	source = sh->source;
-	active_sinks = sh->active_sinks;
-
-	/*
-	 * 4 calls per sink - chown, chmod, utimes, setxattr
-	 */
-	if (xattr)
-		call_count = active_sinks * 4;
-	else
-		call_count = active_sinks * 3;
-
-	local->call_count = call_count;
-
-#ifdef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
-	ts[0] = sh->buf[source].st_atim;
-	ts[1] = sh->buf[source].st_mtim;
-#elif HAVE_STRUCT_STAT_ST_ATIMESPEC_TV_NSEC
-	ts[0] = sh->buf[source].st_atimespec;
-	ts[1] = sh->buf[source].st_mtimespec;
-#else
-	ts[0].tv_sec = sh->buf[source].st_atime;
-	ts[1].tv_sec = sh->buf[source].st_mtime;
-#endif
+        int            i     = 0;
+        afr_private_t *priv  = NULL;
+        afr_local_t   *local = NULL;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (call_count == 0) {
-			break;
-		}
-		if (sh->sources[i] || !local->child_up[i])
-			continue;
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"syncing metadata of %s from %s to %s",
-			local->loc.path, priv->children[source]->name,
-			priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_metadata_attr_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->chown,
-				   &local->loc,
-				   sh->buf[source].st_uid,
-				   sh->buf[source].st_gid);
-
-		STACK_WIND_COOKIE (frame, afr_sh_metadata_attr_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->chmod,
-				   &local->loc, sh->buf[source].st_mode);
-
-		STACK_WIND_COOKIE (frame, afr_sh_metadata_attr_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->utimens,
-				   &local->loc, ts);
-
-		call_count = call_count - 3;
-
-		if (!xattr)
-			continue;
-
-		STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->setxattr,
-				   &local->loc, xattr, 0);
-		call_count--;
-	}
-
-	return 0;
-}
+        priv = this->private;
+        local = frame->local;
 
+        if (op_ret < 0) {
+                afr_sh_metadata_sync_cbk (frame, cookie,
+                                          this, -1, op_errno, xdata);
+                goto out;
+        }
 
-int
-afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              source = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	source = sh->source;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"getxattr of %s failed on subvolume %s (%s). proceeding without xattr",
-			local->loc.path, priv->children[source]->name,
-			strerror (op_errno));
-
-		afr_sh_metadata_sync (frame, this, NULL);
-	} else {
-		dict_del (xattr, AFR_DATA_PENDING);
-		dict_del (xattr, AFR_METADATA_PENDING);
-		dict_del (xattr, AFR_ENTRY_PENDING);
-		afr_sh_metadata_sync (frame, this, xattr);
-	}
-
-	return 0;
-}
+        i = (long) cookie;
 
+        STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
+                           (void *) (long) i,
+                           priv->children[i],
+                           priv->children[i]->fops->setxattr,
+                           &local->loc, local->xattr_req, 0, NULL);
 
-int
-afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              active_sinks = 0;
-	int              source = 0;
-	int              i = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	source = sh->source;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] == 0 && local->child_up[i] == 1) {
-			active_sinks++;
-			sh->success[i] = 1;
-		}
-	}
-	sh->success[source] = 1;
-
-	if (active_sinks == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no active sinks for performing self-heal on file %s",
-			local->loc.path);
-		afr_sh_metadata_finish (frame, this);
-		return 0;
-	}
-	sh->active_sinks = active_sinks;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"syncing metadata of %s from subvolume %s to %d active sinks",
-		local->loc.path, priv->children[source]->name, active_sinks);
-
-	STACK_WIND (frame, afr_sh_metadata_getxattr_cbk,
-		    priv->children[source],
-		    priv->children[source]->fops->getxattr,
-		    &local->loc, NULL);
-
-	return 0;
+ out:
+        return 0;
 }
 
-
-int
-afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this)
+inline void
+afr_prune_special_keys (dict_t *xattr_dict)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              nsources = 0;
-	int              source = 0;
-	int              i = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	afr_sh_build_pending_matrix (sh->pending_matrix, sh->xattr, 
-				     priv->child_count, AFR_METADATA_PENDING);
-
-	afr_sh_print_pending_matrix (sh->pending_matrix, this);
-
-	afr_sh_mark_sources (sh->pending_matrix, sh->sources, 
-			     priv->child_count);
-
-	afr_sh_supress_errenous_children (sh->sources, sh->child_errno,
-					  priv->child_count);
-
-	nsources = afr_sh_source_count (sh->sources, priv->child_count);
-
-	if ((nsources == 0)
-	    && (priv->favorite_child != -1)
-	    && (sh->child_errno[priv->favorite_child] == 0)) {
-
-		gf_log (this->name, GF_LOG_WARNING,
-			"Picking favorite child %s as authentic source to resolve conflicting metadata of %s",
-			priv->children[priv->favorite_child]->name,
-			local->loc.path);
-
-		sh->sources[priv->favorite_child] = 1;
+        dict_del (xattr_dict, GF_SELINUX_XATTR_KEY);
+}
 
-		nsources = afr_sh_source_count (sh->sources,
-						priv->child_count);
-	}
+inline void
+afr_prune_pending_keys (dict_t *xattr_dict, afr_private_t *priv)
+{
+        int i = 0;
 
-	if (nsources == 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Unable to resolve conflicting metadata of %s. "
-			"Please resolve manually by fixing the "
-			"permissions/ownership of %s on your subvolumes. "
-			"You can also consider 'option favorite-child <>'",
-			local->loc.path, local->loc.path);
+        for (; i < priv->child_count; i++) {
+                dict_del (xattr_dict, priv->pending_key[i]);
+        }
+}
 
-		local->govinda_gOvinda = 1;
+int
+afr_sh_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                     dict_t *xdata)
+{
+        int            i     = 0;
+        afr_private_t *priv  = NULL;
+        afr_local_t   *local = NULL;
 
-		afr_sh_metadata_finish (frame, this);
-		return 0;
-	}
+        priv = this->private;
+        local = frame->local;
 
-	source = afr_sh_select_source (sh->sources, priv->child_count);
-	sh->source = source;
+        if (op_ret < 0) {
+                afr_sh_metadata_sync_cbk (frame, cookie,
+                                          this, -1, op_errno, xdata);
+                goto out;
+        }
 
-	/* detect changes not visible through pending flags -- JIC */
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || sh->child_errno[i])
-			continue;
+        afr_prune_pending_keys (xattr, priv);
 
-		if (PERMISSION_DIFFERS (&sh->buf[i], &sh->buf[source]))
-			sh->sources[i] = 0;
+        afr_prune_special_keys (xattr);
 
-		if (OWNERSHIP_DIFFERS (&sh->buf[i], &sh->buf[source]))
-			sh->sources[i] = 0;
-	}
+        i = (long) cookie;
 
-	afr_sh_metadata_sync_prepare (frame, this);
+        /* send removexattr in bulk via xdata */
+        STACK_WIND_COOKIE (frame, afr_sh_removexattr_cbk,
+                           cookie,
+                           priv->children[i],
+                           priv->children[i]->fops->removexattr,
+                           &local->loc, "", xattr);
 
-	return 0;
+ out:
+        return 0;
 }
 
-
 int
-afr_sh_metadata_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			    int32_t op_ret, int32_t op_errno,
-			    inode_t *inode, struct stat *buf, dict_t *xattr)
+afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"path %s on subvolume %s is of mode 0%o",
-				local->loc.path,
-				priv->children[child_index]->name,
-				buf->st_mode);
-
-			sh->buf[child_index] = *buf;
-			if (xattr)
-				sh->xattr[child_index] = dict_ref (xattr);
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"path %s on subvolume %s => -1 (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-
-			sh->child_errno[child_index] = op_errno;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_metadata_fix (frame, this);
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              source = 0;
+        int              active_sinks = 0;
+        int              call_count = 0;
+        int              i = 0;
+
+        struct iatt      stbuf = {0,};
+        int32_t          valid = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        source = sh->source;
+        active_sinks = sh->active_sinks;
+
+        /*
+         * 2 calls per sink - setattr, setxattr
+         */
+        if (xattr) {
+                call_count = active_sinks * 2;
+                local->xattr_req = dict_ref (xattr);
+        } else
+                call_count = active_sinks;
+
+        local->call_count = call_count;
+
+        stbuf.ia_atime = sh->buf[source].ia_atime;
+        stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
+        stbuf.ia_mtime = sh->buf[source].ia_mtime;
+        stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
+
+        stbuf.ia_uid = sh->buf[source].ia_uid;
+        stbuf.ia_gid = sh->buf[source].ia_gid;
+
+        stbuf.ia_type = sh->buf[source].ia_type;
+        stbuf.ia_prot = sh->buf[source].ia_prot;
+
+        valid = GF_SET_ATTR_MODE  |
+                GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
+                GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (call_count == 0) {
+                        break;
+                }
+                if (sh->sources[i] || !local->child_up[i])
+                        continue;
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "self-healing metadata of %s from %s to %s",
+                        local->loc.path, priv->children[source]->name,
+                        priv->children[i]->name);
+
+                STACK_WIND_COOKIE (frame, afr_sh_metadata_setattr_cbk,
+                                   (void *) (long) i,
+                                   priv->children[i],
+                                   priv->children[i]->fops->setattr,
+                                   &local->loc, &stbuf, valid, NULL);
+
+                call_count--;
+
+                if (!xattr)
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_sh_getxattr_cbk,
+                                   (void *) (long) i,
+                                   priv->children[i],
+                                   priv->children[i]->fops->getxattr,
+                                   &local->loc, NULL, NULL);
+                call_count--;
+        }
+
+        return 0;
 }
 
 
 int
-afr_sh_metadata_lookup (call_frame_t *frame, xlator_t *this)
+afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                              dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-	int              call_count = 0;
-	dict_t          *xattr_req = NULL;
-	int              ret = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	call_count = local->child_count;
-	local->call_count = call_count;
-	
-	xattr_req = dict_new();
-	
-	if (xattr_req)
-		ret = dict_set_uint64 (xattr_req, AFR_METADATA_PENDING,
-				       priv->child_count * sizeof(int32_t));
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"looking up %s on %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame, afr_sh_metadata_lookup_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->lookup,
-					   &local->loc, xattr_req);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              source = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        source = sh->source;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "getxattr of %s failed on subvolume %s (%s). proceeding without xattr",
+                        local->loc.path, priv->children[source]->name,
+                        strerror (op_errno));
+
+                afr_sh_metadata_sync (frame, this, NULL);
+        } else {
+                afr_prune_pending_keys (xattr, priv);
+                afr_sh_metadata_sync (frame, this, xattr);
+        }
+
+        return 0;
 }
 
+static void
+afr_set_metadata_sh_info_str (afr_local_t *local, afr_self_heal_t *sh,
+                              xlator_t *this)
+{
+        afr_private_t    *priv = NULL;
+        int              i = 0;
+        char             num[1024] = {0};
+        size_t           len = 0;
+        char             *string = NULL;
+        size_t           off = 0;
+        char             *source_child =  " from source %s to";
+        char             *format = " %s, ";
+        char             *string_msg = " metadata self heal";
+        char             *pending_matrix_str = NULL;
+        int              down_child_present = 0;
+        int              unknown_child_present = 0;
+        char             *down_subvol_1 = " down subvolume is ";
+        char             *unknown_subvol_1 = " unknown subvolume is";
+        char             *down_subvol_2 = " down subvolumes are ";
+        char             *unknown_subvol_2 = " unknown subvolumes are ";
+        int              down_count = 0;
+        int              unknown_count = 0;
+
+        priv = this->private;
+
+        pending_matrix_str = afr_get_pending_matrix_str (sh->pending_matrix,
+                                                         this);
+
+        if (!pending_matrix_str)
+                pending_matrix_str = "";
+
+        len += snprintf (num, sizeof (num), "%s", string_msg);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((sh->source == i) && (local->child_up[i] == 1)) {
+                        len += snprintf (num, sizeof (num), source_child,
+                                         priv->children[i]->name);
+                } else if ((local->child_up[i] == 1) && (sh->sources[i] == 0)) {
+                        len += snprintf (num, sizeof (num), format,
+                                         priv->children[i]->name);
+                } else if (local->child_up[i] == 0) {
+                        len += snprintf (num, sizeof (num), format,
+                                         priv->children[i]->name);
+                        if (!down_child_present)
+                                down_child_present = 1;
+                        down_count++;
+                } else if (local->child_up[i] == -1) {
+                        len += snprintf (num, sizeof (num), format,
+                                         priv->children[i]->name);
+                        if (!unknown_child_present)
+                                unknown_child_present = 1;
+                        unknown_count++;
+                }
+        }
+
+        if (down_child_present) {
+                if (down_count > 1) {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         down_subvol_2);
+                } else {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         down_subvol_1);
+                }
+        }
+        if (unknown_child_present) {
+                if (unknown_count > 1) {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_2);
+                } else {
+                        len += snprintf (num, sizeof (num), "%s",
+                                         unknown_subvol_1);
+                }
+        }
+
+        len ++;
+
+        string = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+        if (!string)
+                return;
+
+        off += snprintf (string + off, len - off, "%s", string_msg);
+        for (i=0; i < priv->child_count; i++) {
+                if ((sh->source == i) && (local->child_up[i] == 1))
+                        off += snprintf (string + off, len - off, source_child,
+                                         priv->children[i]->name);
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if ((local->child_up[i] == 1)&& (sh->sources[i] == 0))
+                        off += snprintf (string + off, len - off, format,
+                                         priv->children[i]->name);
+        }
+
+        if (down_child_present) {
+                if (down_count > 1) {
+                        off += snprintf (string + off, len - off, "%s",
+                                         down_subvol_2);
+                } else {
+                        off += snprintf (string + off, len - off, "%s",
+                                         down_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == 0)
+                        off += snprintf (string + off, len - off, format,
+                                         priv->children[i]->name);
+        }
+
+        if (unknown_child_present) {
+                if (unknown_count > 1) {
+                        off += snprintf (string + off, len - off, "%s",
+                                 unknown_subvol_2);
+                } else {
+                        off += snprintf (string + off, len - off, "%s",
+                                         unknown_subvol_1);
+                }
+        }
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i] == -1)
+                        off += snprintf (string + off, len - off, format,
+                                         priv->children[i]->name);
+        }
+
+        gf_asprintf (&sh->metadata_sh_info, "%s metadata %s,", string,
+                     pending_matrix_str);
+
+        if (pending_matrix_str && strcmp (pending_matrix_str, ""))
+                GF_FREE (pending_matrix_str);
+
+        if (string && strcmp (string, ""))
+                GF_FREE (string);
+}
 
 int
-afr_sh_metadata_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
+afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              child_index = (long) cookie;
-
-	/* TODO: what if lock fails? */
-	
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			sh->op_failed = 1;
-
-			gf_log (this->name,
-				(op_errno == EAGAIN ? GF_LOG_DEBUG : GF_LOG_ERROR),
-				"locking of %s on child %d failed: %s",
-				local->loc.path, child_index,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"inode of %s on child %d locked",
-				local->loc.path, child_index);
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		if (sh->op_failed) {
-			afr_sh_metadata_finish (frame, this);
-			return 0;
-		}
-
-		afr_sh_metadata_lookup (frame, this);
-	}
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              source = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        source = sh->source;
+
+        afr_sh_mark_source_sinks (frame, this);
+        if (sh->active_sinks == 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no active sinks for performing self-heal on file %s",
+                        local->loc.path);
+                afr_sh_metadata_finish (frame, this);
+                return 0;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "syncing metadata of %s from subvolume %s to %d active sinks",
+                local->loc.path, priv->children[source]->name,
+                sh->active_sinks);
+
+        sh->actual_sh_started = _gf_true;
+        afr_set_self_heal_status (sh, AFR_SELF_HEAL_SYNC_BEGIN);
+        afr_set_metadata_sh_info_str (local, sh, this);
+        STACK_WIND (frame, afr_sh_metadata_getxattr_cbk,
+                    priv->children[source],
+                    priv->children[source]->fops->getxattr,
+                    &local->loc, NULL, NULL);
+
+        return 0;
 }
 
 
-int
-afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
+void
+afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-	int              call_count = 0;
-	struct flock     flock = {0, };
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	call_count = local->child_count;
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		flock.l_start   = 0;
-		flock.l_len     = 0;
-		flock.l_type    = F_WRLCK;
-
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"locking %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame, afr_sh_metadata_lk_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->inodelk,
-					   &local->loc, F_SETLK, &flock);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
+        afr_local_t     *local = NULL;
+        afr_self_heal_t *sh = NULL;
+        afr_private_t   *priv = NULL;
+        int              nsources = 0;
+        int              source = 0;
+        int              i = 0;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        priv = this->private;
+
+        if (op_ret < 0) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_FAILED);
+                afr_sh_set_error (sh, op_errno);
+                afr_sh_metadata_finish (frame, this);
+                goto out;
+        }
+        nsources = afr_build_sources (this, sh->xattr, sh->buf,
+                                      sh->pending_matrix, sh->sources,
+                                      sh->success_children,
+                                      AFR_METADATA_TRANSACTION, NULL, _gf_false);
+        if ((nsources == -1)
+            && (priv->favorite_child != -1)
+            && (sh->child_errno[priv->favorite_child] == 0)) {
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Picking favorite child %s as authentic source to resolve conflicting metadata of %s",
+                        priv->children[priv->favorite_child]->name,
+                        local->loc.path);
+
+                sh->sources[priv->favorite_child] = 1;
+
+                nsources = afr_sh_source_count (sh->sources,
+                                                priv->child_count);
+        }
+
+        if (nsources == -1) {
+                afr_sh_print_split_brain_log (sh->pending_matrix, this,
+                                              local->loc.path);
+                afr_set_split_brain (this, sh->inode, SPB, DONT_KNOW);
+                afr_sh_metadata_fail (frame, this);
+                goto out;
+        }
+
+        afr_set_split_brain (this, sh->inode, NO_SPB, DONT_KNOW);
+        if (nsources == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "No self-heal needed for %s",
+                        local->loc.path);
+
+                afr_sh_metadata_finish (frame, this);
+                goto out;
+        }
+
+        source = afr_sh_select_source (sh->sources, priv->child_count);
+
+        if (source == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No active sources found.");
+
+                afr_sh_metadata_finish (frame, this);
+                goto out;
+        }
+
+        sh->source = source;
+
+        /* detect changes not visible through pending flags -- JIC */
+        for (i = 0; i < priv->child_count; i++) {
+                if (i == source || sh->child_errno[i])
+                        continue;
+
+                if (PERMISSION_DIFFERS (&sh->buf[i], &sh->buf[source]))
+                        sh->sources[i] = 0;
+
+                if (OWNERSHIP_DIFFERS (&sh->buf[i], &sh->buf[source]))
+                        sh->sources[i] = 0;
+        }
+
+        if ((!IA_ISREG (sh->buf[source].ia_type)) &&
+            (!IA_ISDIR (sh->buf[source].ia_type))) {
+                afr_reset_children (sh->fresh_children, priv->child_count);
+                afr_get_fresh_children (sh->success_children, sh->sources,
+                                        sh->fresh_children, priv->child_count);
+                afr_inode_set_read_ctx (this, sh->inode, sh->source,
+                                        sh->fresh_children);
+        }
+
+        if (sh->do_metadata_self_heal && priv->metadata_self_heal)
+                afr_sh_metadata_sync_prepare (frame, this);
+        else
+                afr_sh_metadata_finish (frame, this);
+out:
+        return;
 }
 
+int
+afr_sh_metadata_post_nonblocking_inodelk_cbk (call_frame_t *frame,
+                                              xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking metadata "
+                        "inodelks failed for %s.", local->loc.path);
+                gf_log (this->name, GF_LOG_DEBUG, "Metadata self-heal "
+                        "failed for %s.", local->loc.path);
+                afr_sh_metadata_done (frame, this);
+        } else {
+
+                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking metadata "
+                        "inodelks done for %s. Proceeding to FOP",
+                        local->loc.path);
+                afr_sh_common_lookup (frame, this, &local->loc,
+                                      afr_sh_metadata_fix, NULL,
+                                      AFR_LOOKUP_FAIL_CONFLICTS |
+                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
+                                      NULL);
+        }
+
+        return 0;
+}
 
 int
-afr_self_heal_metadata (call_frame_t *frame, xlator_t *this)
+afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t   *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t *priv = this->private;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
+        afr_local_t         *local    = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        int_lock->domain           = this->name;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
+        int_lock->selfheal_lk_type    = AFR_METADATA_SELF_HEAL_LK;
 
+        afr_set_lock_number (frame, this);
 
-	local = frame->local;
-	sh = &local->self_heal;
+        inodelk->flock.l_start = LLONG_MAX - 1;
+        inodelk->flock.l_len   = 0;
+        inodelk->flock.l_type  = F_WRLCK;
+        int_lock->lock_cbk         = afr_sh_metadata_post_nonblocking_inodelk_cbk;
 
-	if (local->need_metadata_self_heal && priv->metadata_self_heal) {
-		afr_sh_metadata_lock (frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"proceeding to data check on %s",
-			local->loc.path);
-		afr_sh_metadata_done (frame, this);
-	}
+        afr_nonblocking_inodelk (frame, this);
+
+        return 0;
+}
 
-	return 0;
+gf_boolean_t
+afr_can_start_metadata_self_heal (afr_self_heal_t *sh, afr_private_t *priv)
+{
+        if (sh->force_confirm_spb)
+                return _gf_true;
+        if (sh->do_metadata_self_heal && priv->metadata_self_heal)
+                return _gf_true;
+        return _gf_false;
 }
 
+int
+afr_self_heal_metadata (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_private_t *priv = this->private;
+        afr_self_heal_t *sh = &local->self_heal;
+
+        local = frame->local;
+        sh = &local->self_heal;
+        sh->sh_type_in_action = AFR_SELF_HEAL_METADATA;
+
+        if (afr_can_start_metadata_self_heal (sh, priv)) {
+                afr_set_self_heal_status (sh, AFR_SELF_HEAL_STARTED);
+                afr_sh_metadata_lock (frame, this);
+        } else {
+                afr_sh_metadata_done (frame, this);
+        }
+
+        return 0;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index 1c97a9bc1..7c9bc8111 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __AFR_SELF_HEAL_H__
@@ -22,19 +13,12 @@
 
 #include <sys/stat.h>
 
-#define FILETYPE_DIFFERS(buf1,buf2) ((S_IFMT & ((struct stat *)buf1)->st_mode) != (S_IFMT & ((struct stat *)buf2)->st_mode))
-#define PERMISSION_DIFFERS(buf1,buf2) ((((struct stat *)buf1)->st_mode) != (((struct stat *)buf2)->st_mode))
-#define OWNERSHIP_DIFFERS(buf1,buf2) ((((struct stat *)buf1)->st_uid) != (((struct stat *)buf2)->st_uid) || (((struct stat *)buf1)->st_gid != (((struct stat *)buf2)->st_gid)))
-#define SIZE_DIFFERS(buf1,buf2) ((((struct stat *)buf1)->st_size) != (((struct stat *)buf2)->st_size))
-
+#define FILETYPE_DIFFERS(buf1,buf2) ((buf1)->ia_type != (buf2)->ia_type)
+#define PERMISSION_DIFFERS(buf1,buf2) (st_mode_from_ia ((buf1)->ia_prot, (buf1)->ia_type) != st_mode_from_ia ((buf2)->ia_prot, (buf2)->ia_type))
+#define OWNERSHIP_DIFFERS(buf1,buf2) (((buf1)->ia_uid != (buf2)->ia_uid) || ((buf1)->ia_gid != (buf2)->ia_gid))
+#define SIZE_DIFFERS(buf1,buf2) ((buf1)->ia_size != (buf2)->ia_size)
 
-
-int
-afr_sh_has_metadata_pending (dict_t *xattr, int child_count, xlator_t *this);
-int
-afr_sh_has_entry_pending (dict_t *xattr, int child_count, xlator_t *this);
-int
-afr_sh_has_data_pending (dict_t *xattr, int child_count, xlator_t *this);
+#define SIZE_GREATER(buf1,buf2) ((buf1)->ia_size > (buf2)->ia_size)
 
 int
 afr_self_heal_entry (call_frame_t *frame, xlator_t *this);
@@ -46,7 +30,14 @@ int
 afr_self_heal_metadata (call_frame_t *frame, xlator_t *this);
 
 int
-afr_self_heal (call_frame_t *frame, xlator_t *this,
-	       int (*completion_cbk) (call_frame_t *, xlator_t *));
+afr_self_heal_get_source (xlator_t *this, afr_local_t *local, dict_t **xattr);
+
+int
+afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode);
 
+int
+afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
+                                          dict_t **xattr,
+                                          afr_transaction_type txn_type,
+                                          uuid_t gfid);
 #endif /* __AFR_SELF_HEAL_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
new file mode 100644
index 000000000..1b48a1bca
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -0,0 +1,1787 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include "afr.h"
+#include "syncop.h"
+#include "afr-self-heald.h"
+#include "afr-self-heal-common.h"
+#include "protocol-common.h"
+#include "event-history.h"
+
+typedef enum {
+        STOP_CRAWL_ON_SINGLE_SUBVOL = 1
+} afr_crawl_flags_t;
+
+typedef enum {
+        HEAL = 1,
+        INFO,
+        STATISTICS_TO_BE_HEALED,
+} shd_crawl_op;
+
+typedef struct shd_dump {
+        dict_t   *dict;
+        xlator_t *this;
+        int      child;
+} shd_dump_t;
+
+typedef struct shd_event_ {
+        int     child;
+        char    *path;
+} shd_event_t;
+
+typedef struct shd_pos_ {
+        int     child;
+        xlator_t *this;
+        afr_child_pos_t pos;
+} shd_pos_t;
+
+typedef int
+(*afr_crawl_done_cbk_t)  (int ret, call_frame_t *sync_frame, void *crawl_data);
+
+void
+afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
+                 process_entry_cbk_t process_entry, void *op_data,
+                 gf_boolean_t exclusive, int crawl_flags,
+                 afr_crawl_done_cbk_t crawl_done);
+
+static int
+_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data);
+
+/* For calling straight through (e.g. already in a synctask). */
+int
+afr_find_child_position (xlator_t *this, int child, afr_child_pos_t *pos);
+
+/* For deferring through a new synctask. */
+int
+afr_syncop_find_child_position (void *data);
+
+static int
+_loc_assign_gfid_path (loc_t *loc)
+{
+        int  ret = -1;
+        char gfid_path[64] = {0};
+
+        if (loc->inode && !uuid_is_null (loc->inode->gfid)) {
+                ret = inode_path (loc->inode, NULL, (char**)&loc->path);
+        } else if (!uuid_is_null (loc->gfid)) {
+                snprintf (gfid_path, sizeof (gfid_path), "<gfid:%s>",
+                          uuid_utoa (loc->gfid));
+                loc->path = gf_strdup (gfid_path);
+                if (loc->path)
+                        ret = 0;
+        }
+        return ret;
+}
+
+void
+_destroy_crawl_event_data (void *data)
+{
+        shd_crawl_event_t        *crawl_event = NULL;
+
+        if (!data)
+                goto out;
+
+        crawl_event = (shd_crawl_event_t *)data;
+        GF_FREE (crawl_event->start_time_str);
+        GF_FREE (crawl_event->end_time_str);
+
+out:
+        return;
+}
+
+void
+_destroy_shd_event_data (void *data)
+{
+        shd_event_t             *event = NULL;
+        if (!data)
+                goto out;
+        event = (shd_event_t*)data;
+        GF_FREE (event->path);
+out:
+        return;
+}
+void
+shd_cleanup_event (void *event)
+{
+        shd_event_t *shd_event = event;
+
+        if (!shd_event)
+                goto out;
+        GF_FREE (shd_event->path);
+        GF_FREE (shd_event);
+out:
+        return;
+}
+
+int
+afr_get_local_child (afr_self_heald_t *shd, unsigned int child_count)
+{
+        int i = 0;
+        int ret = -1;
+        for (i = 0; i < child_count; i++) {
+                if (shd->pos[i] == AFR_POS_LOCAL) {
+                        ret = i;
+                        break;
+                }
+        }
+        return ret;
+}
+
+static int
+_build_index_loc (xlator_t *this, loc_t *loc, char *name, loc_t *parent)
+{
+        int             ret = 0;
+
+        uuid_copy (loc->pargfid, parent->inode->gfid);
+        loc->path = "";
+        loc->name = name;
+        loc->parent = inode_ref (parent->inode);
+        if (!loc->parent) {
+                loc->path = NULL;
+                loc_wipe (loc);
+                ret = -1;
+        }
+        return ret;
+}
+
+int
+_add_crawl_stats_to_dict (xlator_t *this, dict_t *output, int child,
+                          shd_crawl_event_t *shd_event, struct timeval *tv)
+{
+        int             ret = 0;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+        uint64_t        healed_count = 0;
+        uint64_t        split_brain_count = 0;
+        uint64_t        heal_failed_count = 0;
+        char            *start_time_str = NULL;
+        char            *end_time_str = NULL;
+        char            *crawl_type = NULL;
+        int             progress = -1;
+
+        healed_count = shd_event->healed_count;
+        split_brain_count = shd_event->split_brain_count;
+        heal_failed_count = shd_event->heal_failed_count;
+        start_time_str = shd_event->start_time_str;
+        end_time_str = shd_event->end_time_str;
+        crawl_type = shd_event->crawl_type;
+
+        if (!start_time_str) {
+                ret = -1;
+                goto out;
+        }
+
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64(output, key, healed_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "healed_count to outout");
+                goto out;
+         }
+        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, split_brain_count);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "split_brain_count to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, gf_strdup (crawl_type));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_type to output");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, heal_failed_count);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "healed_failed_count to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, gf_strdup(start_time_str));
+
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_start_time to outout");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        if (!end_time_str)
+                end_time_str = "Could not determine the end time";
+        ret = dict_set_dynstr (output, key, gf_strdup(end_time_str));
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "crawl_end_time to outout");
+                goto out;
+        }
+        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        if (shd_event->crawl_inprogress == _gf_true)
+                progress = 1;
+        else
+                progress = 0;
+
+        ret = dict_set_int32 (output, key, progress);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not add statistics_"
+                        "inprogress to outout");
+                goto out;
+        }
+
+         snprintf (key, sizeof (key), "statistics-%d-%d-count",xl_id, child);
+         ret = dict_set_uint64 (output, key, count + 1);
+         if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not increment the "
+                        "counter.");
+                goto out;
+         }
+out:
+        return ret;
+}
+
+int
+_add_path_to_dict (xlator_t *this, dict_t *output, int child, char *path,
+                   struct timeval *tv, gf_boolean_t dyn)
+{
+        //subkey not used for now
+        int             ret = -1;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
+        if (dyn)
+                ret = dict_set_dynstr (output, key, path);
+        else
+                ret = dict_set_str (output, key, path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
+                        path);
+                goto out;
+        }
+
+        if (!tv)
+                goto inc_count;
+        snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
+                  child, count);
+        ret = dict_set_uint32 (output, key, tv->tv_sec);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Could not set time",
+                        path);
+                goto out;
+        }
+
+inc_count:
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+        ret = dict_set_uint64 (output, key, count + 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not increment count");
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+_get_path_from_gfid_loc (xlator_t *this, xlator_t *readdir_xl, loc_t *child,
+                         char **fpath, gf_boolean_t *missing)
+{
+        dict_t          *xattr = NULL;
+        char            *path = NULL;
+        int             ret = -1;
+
+        ret = syncop_getxattr (readdir_xl, child, &xattr, GFID_TO_PATH_KEY);
+        if (ret < 0) {
+                if ((errno == ENOENT) && missing)
+                        *missing = _gf_true;
+                goto out;
+        }
+        ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get path for "
+                        "gfid %s", uuid_utoa (child->gfid));
+                goto out;
+        }
+        path = gf_strdup (path);
+        if (!path) {
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        if (!ret)
+                *fpath = path;
+        if (xattr)
+                dict_unref (xattr);
+        return ret;
+}
+
+int
+_add_event_to_dict (circular_buffer_t *cb, void *data)
+{
+        int               ret = 0;
+        shd_dump_t        *dump_data = NULL;
+        shd_event_t       *shd_event = NULL;
+
+        dump_data = data;
+        shd_event = cb->data;
+        if (shd_event->child != dump_data->child)
+                goto out;
+        ret = _add_path_to_dict (dump_data->this, dump_data->dict,
+                                 dump_data->child, shd_event->path, &cb->tv,
+                                 _gf_false);
+out:
+        return ret;
+}
+
+int
+_add_crawl_event_statistics_to_dict (circular_buffer_t *cb, void *data)
+{
+        int               ret = 0;
+        shd_dump_t        *dump_data = NULL;
+        shd_crawl_event_t *shd_event = NULL;
+
+        dump_data = data;
+        shd_event = cb->data;
+        ret = _add_crawl_stats_to_dict (dump_data->this, dump_data->dict,
+                                        dump_data->child, shd_event, &cb->tv);
+        return ret;
+}
+
+int
+_add_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict, int child)
+{
+        shd_dump_t dump_data = {0};
+
+        dump_data.this = this;
+        dump_data.dict = dict;
+        dump_data.child = child;
+        eh_dump (eh, &dump_data, _add_event_to_dict);
+        return 0;
+}
+
+
+int
+_add_statistics_to_dict (xlator_t *this, dict_t *dict, int child)
+{
+        shd_dump_t              dump_data = {0};
+        afr_private_t           *priv  = NULL;
+        afr_self_heald_t        *shd = NULL;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        dump_data.this = this;
+        dump_data.dict = dict;
+        dump_data.child = child;
+        eh_dump (shd->statistics[child], &dump_data,
+                 _add_crawl_event_statistics_to_dict);
+        return 0;
+
+}
+
+void
+_remove_stale_index (xlator_t *this, xlator_t *readdir_xl,
+                     loc_t *parent, char *fname)
+{
+        int              ret = 0;
+        loc_t            index_loc = {0};
+
+        ret = _build_index_loc (this, &index_loc, fname, parent);
+        if (ret)
+                goto out;
+        gf_log (this->name, GF_LOG_DEBUG, "Removing stale index "
+                "for %s on %s", index_loc.name, readdir_xl->name);
+        ret = syncop_unlink (readdir_xl, &index_loc);
+        if(ret && (errno != ENOENT)) {
+                gf_log(this->name, GF_LOG_ERROR, "%s: Failed to remove index "
+                       "on %s - %s",index_loc.name, readdir_xl->name,
+                       strerror (errno));
+        }
+        index_loc.path = NULL;
+        loc_wipe (&index_loc);
+out:
+        return;
+}
+
+int
+_count_hard_links_under_base_indices_dir (xlator_t *this,
+                                           afr_crawl_data_t *crawl_data,
+                                           gf_dirent_t *entry, loc_t *childloc,
+                                           loc_t *parentloc, struct iatt *iattr)
+{
+        xlator_t                *readdir_xl = crawl_data->readdir_xl;
+        struct iatt             parent = {0};
+        int                     ret = 0;
+        dict_t                  *output = NULL;
+        int                     xl_id =  0;
+        char                    key[256] = {0};
+        int                     child  = -1;
+        uint64_t                     hardlinks = 0;
+
+        output = crawl_data->op_data;
+        child = crawl_data->child;
+
+        ret = syncop_lookup (readdir_xl, childloc, NULL, iattr, NULL, &parent);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret)
+                goto out;
+
+        snprintf (key, sizeof (key), "%d-%d-hardlinks", xl_id, child);
+        ret =  dict_get_uint64 (output, key, &hardlinks);
+
+        /*Removing the count of base_entry under indices/base_indicies and
+         * entry under indices/xattrop */
+        hardlinks = hardlinks + iattr->ia_nlink - 2;
+        ret = dict_set_uint64 (output, key, hardlinks);
+        if (ret)
+                goto out;
+
+out:
+        return ret;
+}
+
+int
+_add_summary_to_dict (xlator_t *this, afr_crawl_data_t *crawl_data,
+                      gf_dirent_t *entry,
+                      loc_t *childloc, loc_t *parentloc, struct iatt *iattr)
+{
+        dict_t          *output = NULL;
+        xlator_t        *readdir_xl = NULL;
+        int             ret = -1;
+        char            *path = NULL;
+        gf_boolean_t    missing = _gf_false;
+        char            gfid_str[64] = {0};
+
+        if (uuid_is_null (childloc->gfid))
+                goto out;
+
+        output = crawl_data->op_data;
+        readdir_xl = crawl_data->readdir_xl;
+
+        ret = _get_path_from_gfid_loc (this, readdir_xl, childloc, &path,
+                                       &missing);
+        if (ret == 0) {
+                ret = _add_path_to_dict (this, output, crawl_data->child, path,
+                                         NULL, _gf_true);
+        } else if (missing) {
+                _remove_stale_index (this, readdir_xl, parentloc,
+                                     uuid_utoa_r (childloc->gfid, gfid_str));
+        }
+
+out:
+        if (ret && path)
+                GF_FREE (path);
+        return ret;
+}
+
+void
+_crawl_post_sh_action (xlator_t *this, loc_t *parent, loc_t *child,
+                       int32_t op_ret, int32_t op_errno, dict_t *xattr_rsp,
+                       afr_crawl_data_t *crawl_data)
+{
+        int                ret = 0;
+        afr_private_t      *priv = NULL;
+        afr_self_heald_t   *shd = NULL;
+        eh_t               *eh = NULL;
+        char               *path = NULL;
+        char               gfid_str[64] = {0};
+        shd_event_t        *event = NULL;
+        int32_t            sh_failed = 0;
+        gf_boolean_t       split_brain = 0;
+        int32_t            actual_sh_done = 0;
+        shd_crawl_event_t  **shd_crawl_event = NULL;
+
+        priv = this->private;
+        shd  = &priv->shd;
+        if (crawl_data->crawl == INDEX) {
+                if ((op_ret < 0) && (op_errno == ENOENT)) {
+                        _remove_stale_index (this, crawl_data->readdir_xl,
+                                             parent, uuid_utoa_r (child->gfid,
+                                                                  gfid_str));
+                        goto out;
+                }
+                ret = _get_path_from_gfid_loc (this, crawl_data->readdir_xl,
+                                               child, &path, NULL);
+                if (ret)
+                        goto out;
+        } else {
+                path = gf_strdup (child->path);
+                if (!path) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (xattr_rsp) {
+                ret = dict_get_int32 (xattr_rsp, "sh-failed", &sh_failed);
+                ret = dict_get_int32 (xattr_rsp, "actual-sh-done", &actual_sh_done);
+        }
+
+        shd_crawl_event = (shd_crawl_event_t**)(shd->crawl_events);
+
+        split_brain = afr_is_split_brain (this, child->inode);
+        if ((op_ret < 0 && op_errno == EIO) || split_brain) {
+                eh = shd->split_brain;
+                shd_crawl_event[crawl_data->child]->split_brain_count += 1;
+        } else if ((op_ret < 0) || sh_failed) {
+                eh = shd->heal_failed;
+                shd_crawl_event[crawl_data->child]->heal_failed_count += 1;
+        } else if (actual_sh_done == 1) {
+                eh = shd->healed;
+                shd_crawl_event[crawl_data->child]->healed_count += 1;
+        }
+        ret = -1;
+
+        if (eh != NULL) {
+                event = GF_CALLOC (1, sizeof (*event), gf_afr_mt_shd_event_t);
+                if (!event)
+                        goto out;
+                event->child = crawl_data->child;
+                event->path = path;
+
+                ret = eh_save_history (eh, event);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s:Failed to save "
+                                "to event history, (%d, %s)", path, op_ret,
+                                strerror (op_errno));
+
+                        goto out;
+                }
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG, "%s:Self heal already done ",
+                        path);
+
+        }
+        ret = 0;
+out:
+        if (ret && path)
+                GF_FREE (path);
+        return;
+}
+
+int
+_link_inode_update_loc (xlator_t *this, loc_t *loc, struct iatt *iattr)
+{
+        inode_t       *link_inode = NULL;
+        int           ret = -1;
+
+        link_inode = inode_link (loc->inode, NULL, NULL, iattr);
+        if (link_inode == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "inode link failed "
+                        "on the inode (%s)", uuid_utoa (iattr->ia_gfid));
+                goto out;
+        }
+        inode_unref (loc->inode);
+        loc->inode = link_inode;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+_self_heal_entry (xlator_t *this, afr_crawl_data_t *crawl_data, gf_dirent_t *entry,
+                  loc_t *child, loc_t *parent, struct iatt *iattr)
+{
+        struct iatt      parentbuf = {0};
+        int              ret = 0;
+        dict_t           *xattr_rsp = NULL;
+        dict_t           *xattr_req = NULL;
+
+        xattr_req = dict_new ();
+        if (!xattr_req) {
+                errno = ENOMEM;
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_int32 (xattr_req, "allow-sh-for-running-transaction", 1);
+
+        gf_log (this->name, GF_LOG_DEBUG, "lookup %s", child->path);
+
+        ret = syncop_lookup (this, child, xattr_req,
+                             iattr, &xattr_rsp, &parentbuf);
+        _crawl_post_sh_action (this, parent, child, ret, errno, xattr_rsp,
+                               crawl_data);
+        if (xattr_rsp)
+                dict_unref (xattr_rsp);
+        if (ret == 0)
+                ret = _link_inode_update_loc (this, child, iattr);
+
+out:
+        if (xattr_req)
+                dict_unref(xattr_req);
+        return ret;
+}
+
+static int
+afr_crawl_done  (int ret, call_frame_t *sync_frame, void *data)
+{
+        GF_FREE (data);
+        STACK_DESTROY (sync_frame->root);
+        return 0;
+}
+
+void
+_do_self_heal_on_subvol (xlator_t *this, int child, afr_crawl_type_t crawl)
+{
+        afr_start_crawl (this, child, crawl, _self_heal_entry,
+                         NULL, _gf_true, STOP_CRAWL_ON_SINGLE_SUBVOL,
+                         afr_crawl_done);
+}
+
+gf_boolean_t
+_crawl_proceed (xlator_t *this, int child, int crawl_flags, char **reason)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        gf_boolean_t            proceed = _gf_false;
+        char                    *msg = NULL;
+
+        priv = this->private;
+        shd  = &priv->shd;
+        if (!shd->enabled) {
+                msg = "Self-heal daemon is not enabled";
+                gf_log (this->name, GF_LOG_DEBUG, "%s", msg);
+                goto out;
+        }
+        if (!priv->child_up[child]) {
+                gf_log (this->name, GF_LOG_DEBUG, "Stopping crawl for %s , "
+                        "subvol went down", priv->children[child]->name);
+                msg = "Brick is Not connected";
+                goto out;
+        }
+
+        if (crawl_flags & STOP_CRAWL_ON_SINGLE_SUBVOL) {
+                if (afr_up_children_count (priv->child_up,
+                                           priv->child_count) < 2) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Stopping crawl as "
+                                "< 2 children are up");
+                        msg = "< 2 bricks in replica are running";
+                        goto out;
+                }
+        }
+        proceed = _gf_true;
+out:
+        if (reason)
+                *reason = msg;
+        return proceed;
+}
+
+int
+_do_crawl_op_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
+                               shd_crawl_op op, dict_t *output)
+{
+        afr_private_t       *priv = NULL;
+        char                *status = NULL;
+        char                *subkey = NULL;
+        char                key[256] = {0};
+        shd_pos_t           pos_data = {0};
+        int                 op_ret = -1;
+        int                 xl_id = -1;
+        int                 i = 0;
+        int                 ret = 0;
+        int                 crawl_flags = 0;
+
+        priv = this->private;
+        if (op == HEAL)
+                crawl_flags |= STOP_CRAWL_ON_SINGLE_SUBVOL;
+
+        if (output) {
+                ret = dict_get_int32 (output, this->name, &xl_id);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid input, "
+                                "translator-id is not available");
+                        goto out;
+                }
+        }
+        pos_data.this = this;
+        subkey = "status";
+        for (i = 0; i < priv->child_count; i++) {
+                if (_crawl_proceed (this, i, crawl_flags, &status)) {
+                        pos_data.child = i;
+                        /*
+                         * We're already in a synctask in this case, so we
+                         * don't need to defer through a second (and in fact
+                         * that can cause deadlock).  Just call straight
+                         * through instead.
+                         */
+                        ret = afr_find_child_position(pos_data.this,
+                                                      pos_data.child,
+                                                      &pos_data.pos);
+                        if (ret) {
+                                status = "Not able to find brick location";
+                        } else if (pos_data.pos == AFR_POS_REMOTE) {
+                                status = "brick is remote";
+                        } else {
+                                op_ret = 0;
+                                if (op == HEAL) {
+                                        status = "Started self-heal";
+                                        _do_self_heal_on_subvol (this, i,
+                                                                 crawl);
+                                } else if (output && (op == INFO)) {
+                                        status = "";
+                                        afr_start_crawl (this, i, INDEX,
+                                                         _add_summary_to_dict,
+                                                         output, _gf_false, 0,
+                                                         NULL);
+                                } else if (output &&
+                                           (op == STATISTICS_TO_BE_HEALED)) {
+                                            status = "";
+                                            afr_start_crawl (this, i,
+                                                             INDEX_TO_BE_HEALED,
+                                       _count_hard_links_under_base_indices_dir,
+                                                             output, _gf_false,
+                                                             0, NULL);
+                                }
+                        }
+                        if (output) {
+                                snprintf (key, sizeof (key), "%d-%d-%s", xl_id,
+                                          i, subkey);
+                                ret = dict_set_str (output, key, status);
+                        }
+                        if (!op_ret && (crawl == FULL))
+                                break;
+                }
+                if (output) {
+                        snprintf (key, sizeof (key), "%d-%d-%s", xl_id, i,
+                                  subkey);
+                        ret = dict_set_str (output, key, status);
+                }
+        }
+out:
+        return op_ret;
+}
+
+int
+_do_self_heal_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
+                                dict_t *output)
+{
+        return _do_crawl_op_on_local_subvols (this, crawl, HEAL, output);
+}
+
+int
+_get_index_summary_on_local_subvols (xlator_t *this, dict_t *output)
+{
+        return _do_crawl_op_on_local_subvols (this, INDEX, INFO, output);
+}
+
+void
+afr_fill_completed_crawl_statistics_to_dict (xlator_t *this, dict_t *dict)
+{
+        afr_private_t           *priv  = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+        priv = this->private;
+        shd= &priv->shd;
+        for (i = 0; i < priv->child_count; i++) {
+                if (shd->pos[i] != AFR_POS_LOCAL)
+                        continue;
+                _add_statistics_to_dict (this, dict, i);
+        }
+
+        return ;
+}
+
+static void
+reset_crawl_event (shd_crawl_event_t *crawl_event)
+{
+    crawl_event->healed_count = 0;
+    crawl_event->split_brain_count = 0;
+    crawl_event->heal_failed_count = 0;
+    GF_FREE (crawl_event->start_time_str);
+    crawl_event->start_time_str = NULL;
+    crawl_event->end_time_str = NULL;
+    crawl_event->crawl_type = NULL;
+    crawl_event->crawl_inprogress = _gf_false;
+    return;
+}
+
+static void
+afr_copy_crawl_event_struct (shd_crawl_event_t *src, shd_crawl_event_t *dst)
+{
+        dst->healed_count = src->healed_count;
+        dst->split_brain_count = src->split_brain_count;
+        dst->heal_failed_count = src->heal_failed_count;
+        dst->start_time_str = gf_strdup (src->start_time_str);
+        dst->end_time_str = "Crawl is already in progress";
+        dst->crawl_type = src->crawl_type;
+        dst->crawl_inprogress = _gf_true;
+        return;
+}
+
+static int
+afr_fill_crawl_statistics_of_running_crawl(xlator_t *this, dict_t *dict)
+{
+        shd_crawl_event_t       *evnt = NULL;
+        int                     ret = 0;
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+        priv = this->private;
+        shd = &priv->shd;
+
+        evnt = GF_CALLOC (1, sizeof (shd_crawl_event_t),
+                          gf_afr_mt_shd_crawl_event_t);
+        if (!evnt) {
+                ret = -1;
+                goto out;
+        }
+        LOCK (&priv->lock);
+        {
+                for (i = 0; i < priv->child_count; i++) {
+                        if (shd->pos[i] != AFR_POS_LOCAL)
+                                continue;
+
+                        reset_crawl_event (evnt);
+
+                        if (!shd->crawl_events[i]) {
+                                continue;
+                        }
+
+                        afr_copy_crawl_event_struct (shd->crawl_events[i],
+                                                     evnt);
+                        _add_crawl_stats_to_dict (this, dict, i, evnt, NULL);
+
+                }
+        }
+        UNLOCK (&priv->lock);
+        reset_crawl_event (evnt);
+        GF_FREE (evnt);
+
+out:
+        return ret;
+}
+
+static int
+_add_local_subvols_crawl_statistics_to_dict (xlator_t *this, dict_t *dict)
+{
+        int ret = 0;
+        afr_fill_completed_crawl_statistics_to_dict (this, dict);
+        ret = afr_fill_crawl_statistics_of_running_crawl (this, dict);
+        return ret;
+}
+int
+_add_local_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     i = 0;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (shd->pos[i] != AFR_POS_LOCAL)
+                        continue;
+                _add_eh_to_dict (this, eh, dict, i);
+        }
+        return 0;
+}
+
+int
+afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+{
+        gf_xl_afr_op_t   op = GF_AFR_OP_INVALID;
+        int              ret = 0;
+        afr_private_t    *priv = NULL;
+        afr_self_heald_t *shd = NULL;
+        int              xl_id = 0;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (input, this->name, &xl_id);
+        if (ret)
+                goto out;
+        ret = dict_set_int32 (output, this->name, xl_id);
+        if (ret)
+                goto out;
+        switch (op) {
+        case GF_AFR_OP_HEAL_INDEX:
+                ret = _do_self_heal_on_local_subvols (this, INDEX, output);
+                break;
+        case GF_AFR_OP_HEAL_FULL:
+                ret = _do_self_heal_on_local_subvols (this, FULL, output);
+                break;
+        case GF_AFR_OP_INDEX_SUMMARY:
+                (void)_get_index_summary_on_local_subvols (this, output);
+                ret = 0;
+                break;
+        case GF_AFR_OP_HEALED_FILES:
+                ret = _add_local_subvols_eh_to_dict (this, shd->healed, output);
+                break;
+        case GF_AFR_OP_HEAL_FAILED_FILES:
+                ret = _add_local_subvols_eh_to_dict (this, shd->heal_failed,
+                                                   output);
+                break;
+        case GF_AFR_OP_SPLIT_BRAIN_FILES:
+                ret = _add_local_subvols_eh_to_dict (this, shd->split_brain,
+                                                   output);
+                break;
+        case GF_AFR_OP_STATISTICS:
+                ret = _add_local_subvols_crawl_statistics_to_dict (this, output);
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                ret = _do_crawl_op_on_local_subvols (this, INDEX_TO_BE_HEALED,
+                                                     STATISTICS_TO_BE_HEALED,
+                                                     output);
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
+                break;
+        }
+out:
+        dict_del (output, this->name);
+        return ret;
+}
+
+void
+afr_poll_self_heal (void *data)
+{
+        afr_private_t    *priv = NULL;
+        afr_self_heald_t *shd = NULL;
+        struct timespec  timeout = {0};
+        xlator_t         *this = NULL;
+        long             child = (long)data;
+        gf_timer_t       *old_timer = NULL;
+        gf_timer_t       *new_timer = NULL;
+        shd_pos_t        pos_data = {0};
+        int              ret = 0;
+
+        this = THIS;
+        priv = this->private;
+        shd = &priv->shd;
+
+        if (shd->pos[child] == AFR_POS_UNKNOWN) {
+                pos_data.this = this;
+                pos_data.child = child;
+                ret = synctask_new (this->ctx->env,
+                                    afr_syncop_find_child_position,
+                                    NULL, NULL, &pos_data);
+                if (!ret)
+                        shd->pos[child] = pos_data.pos;
+        }
+        if (shd->enabled && (shd->pos[child] == AFR_POS_LOCAL))
+                _do_self_heal_on_subvol (this, child, INDEX);
+        timeout.tv_sec = shd->timeout;
+        timeout.tv_nsec = 0;
+        //notify and previous timer should be synchronized.
+        LOCK (&priv->lock);
+        {
+                old_timer = shd->timer[child];
+                if (shd->pos[child] == AFR_POS_REMOTE)
+                        goto unlock;
+                shd->timer[child] = gf_timer_call_after (this->ctx, timeout,
+                                                         afr_poll_self_heal,
+                                                         data);
+                new_timer = shd->timer[child];
+        }
+unlock:
+        UNLOCK (&priv->lock);
+
+        if (old_timer)
+                gf_timer_call_cancel (this->ctx, old_timer);
+        if (!new_timer && (shd->pos[child] != AFR_POS_REMOTE)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not create self-heal polling timer for %s",
+                        priv->children[child]->name);
+        }
+        return;
+}
+
+static int
+afr_handle_child_up  (int ret, call_frame_t *sync_frame, void *data)
+{
+        afr_self_heald_t *shd = NULL;
+        shd_pos_t        *pos_data = data;
+        afr_private_t    *priv = NULL;
+
+        if (ret)
+                goto out;
+
+        priv = pos_data->this->private;
+        shd = &priv->shd;
+        shd->pos[pos_data->child] = pos_data->pos;
+        if (pos_data->pos != AFR_POS_REMOTE)
+                afr_poll_self_heal ((void*)(long)pos_data->child);
+        _do_self_heal_on_local_subvols (THIS, INDEX, NULL);
+out:
+        GF_FREE (data);
+        return 0;
+}
+
+void
+afr_proactive_self_heal (void *data)
+{
+        xlator_t         *this = NULL;
+        long             child = (long)data;
+        shd_pos_t        *pos_data = NULL;
+        int              ret = 0;
+
+        this = THIS;
+
+        //Position of brick could have changed and it could be local now.
+        //Compute the position again
+        pos_data = GF_CALLOC (1, sizeof (*pos_data), gf_afr_mt_pos_data_t);
+        if (!pos_data)
+                goto out;
+        pos_data->this = this;
+        pos_data->child = child;
+        ret = synctask_new (this->ctx->env, afr_syncop_find_child_position,
+                            afr_handle_child_up, NULL, pos_data);
+        if (ret)
+                goto out;
+out:
+        return;
+}
+
+static int
+get_pathinfo_host (char *pathinfo, char *hostname, size_t size)
+{
+        char    *start = NULL;
+        char    *end = NULL;
+        int     ret  = -1;
+        int     i    = 0;
+
+        if (!pathinfo)
+                goto out;
+
+        start = strchr (pathinfo, ':');
+        if (!start)
+                goto out;
+        end = strrchr (pathinfo, ':');
+        if (start == end)
+                goto out;
+
+        memset (hostname, 0, size);
+        i = 0;
+        while (++start != end)
+                hostname[i++] = *start;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *local)
+{
+        int             ret   = 0;
+        char            pathinfohost[1024] = {0};
+        char            localhost[1024] = {0};
+        xlator_t        *this = THIS;
+
+        *local = _gf_false;
+        ret = get_pathinfo_host (pathinfo, pathinfohost, sizeof (pathinfohost));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid pathinfo: %s",
+                        pathinfo);
+                goto out;
+        }
+
+        ret = gethostname (localhost, sizeof (localhost));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "gethostname() failed, "
+                        "reason: %s", strerror (errno));
+                goto out;
+        }
+
+        if (!strcmp (localhost, pathinfohost))
+                *local = _gf_true;
+out:
+        return ret;
+}
+
+int
+afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
+                           loc_t *dirloc)
+{
+        afr_private_t *priv = NULL;
+        dict_t        *xattr = NULL;
+        void          *index_gfid = NULL;
+        void          *base_indices_holder_vgfid = NULL;
+        loc_t         rootloc = {0};
+        struct iatt   iattr = {0};
+        struct iatt   parent = {0};
+        int           ret = 0;
+        xlator_t      *readdir_xl = crawl_data->readdir_xl;
+
+        priv = this->private;
+        if (crawl_data->crawl == FULL) {
+                afr_build_root_loc (this, dirloc);
+        } else if (crawl_data->crawl == INDEX) {
+                afr_build_root_loc (this, &rootloc);
+                ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
+                                       GF_XATTROP_INDEX_GFID);
+                if (ret < 0)
+                        goto out;
+                ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to get index "
+                                "dir gfid on %s", readdir_xl->name);
+                        goto out;
+                }
+                if (!index_gfid) {
+                        gf_log (this->name, GF_LOG_ERROR, "index gfid empty "
+                                "on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                uuid_copy (dirloc->gfid, index_gfid);
+                dirloc->path = "";
+                dirloc->inode = inode_new (priv->root_inode->table);
+                ret = syncop_lookup (readdir_xl, dirloc, NULL,
+                                     &iattr, NULL, &parent);
+                if (ret < 0) {
+                        if (errno != ENOENT) {
+                                gf_log (this->name, GF_LOG_ERROR, "lookup "
+                                        "failed on index dir on %s - (%s)",
+                                        readdir_xl->name, strerror (errno));
+                        }
+                        goto out;
+                }
+                ret = _link_inode_update_loc (this, dirloc, &iattr);
+                if (ret)
+                        goto out;
+        } else if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                afr_build_root_loc (this, &rootloc);
+                ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
+                                       GF_BASE_INDICES_HOLDER_GFID);
+                if (ret < 0)
+                        goto out;
+                ret = dict_get_ptr (xattr, GF_BASE_INDICES_HOLDER_GFID,
+                                    &base_indices_holder_vgfid);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "index gfid empty "
+                                "on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                if (!base_indices_holder_vgfid) {
+                        gf_log (this->name, GF_LOG_ERROR, "Base indices holder"
+                                "virtual gfid is null on %s", readdir_xl->name);
+                        ret = -1;
+                        goto out;
+                }
+                uuid_copy (dirloc->gfid,  base_indices_holder_vgfid);
+                dirloc->path = "";
+                dirloc->inode = inode_new (priv->root_inode->table);
+                ret = syncop_lookup (readdir_xl, dirloc, NULL, &iattr, NULL,
+                                     &parent);
+                if (ret < 0) {
+                        if (errno != ENOENT) {
+                                gf_log (this->name, GF_LOG_ERROR, "lookup "
+                                        "failed for base_indices_holder dir"
+                                        " on %s - (%s)", readdir_xl->name,
+                                        strerror (errno));
+
+                        } else {
+                                gf_log (this->name, GF_LOG_ERROR, "base_indices"
+                                        "_holder is not yet created.");
+                        }
+                        goto out;
+                }
+                ret = _link_inode_update_loc (this, dirloc, &iattr);
+                if (ret)
+                        goto out;
+        }
+        ret = 0;
+out:
+        if (xattr)
+                dict_unref (xattr);
+        loc_wipe (&rootloc);
+        return ret;
+}
+
+int
+afr_crawl_opendir (xlator_t *this, afr_crawl_data_t *crawl_data, fd_t **dirfd,
+                   loc_t *dirloc)
+{
+        fd_t          *fd   = NULL;
+        int           ret = 0;
+
+        if (crawl_data->crawl == FULL) {
+                fd = fd_create (dirloc->inode, crawl_data->pid);
+                if (!fd) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to create fd for %s", dirloc->path);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = syncop_opendir (crawl_data->readdir_xl, dirloc, fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "opendir failed on %s", dirloc->path);
+                        goto out;
+                }
+        } else {
+                fd = fd_anonymous (dirloc->inode);
+        }
+        ret = 0;
+out:
+        if (!ret)
+                *dirfd = fd;
+        return ret;
+}
+
+xlator_t*
+afr_crawl_readdir_xl_get (xlator_t *this, afr_crawl_data_t *crawl_data)
+{
+        afr_private_t *priv = this->private;
+
+        if (crawl_data->crawl == FULL) {
+                return this;
+        } else {
+                return priv->children[crawl_data->child];
+        }
+        return NULL;
+}
+
+int
+afr_crawl_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent,
+                           gf_dirent_t *entry, afr_crawl_data_t *crawl_data)
+{
+        int           ret = -1;
+        afr_private_t *priv = NULL;
+
+        priv = this->private;
+        if (crawl_data->crawl == FULL) {
+                ret = afr_build_child_loc (this, child, parent, entry->d_name);
+        } else if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                ret = _build_index_loc (this, child, entry->d_name, parent);
+                if (ret)
+                        goto out;
+                child->inode = inode_new (priv->root_inode->table);
+                if (!child->inode) {
+                        ret = -1;
+                        goto out;
+                }
+                child->path = NULL;
+        } else {
+                child->inode = inode_new (priv->root_inode->table);
+                if (!child->inode)
+                        goto out;
+                uuid_parse (entry->d_name, child->gfid);
+                ret = _loc_assign_gfid_path (child);
+        }
+out:
+        return ret;
+}
+
+static int
+_process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
+                  off_t *offset, afr_crawl_data_t *crawl_data)
+{
+        gf_dirent_t      *entry = NULL;
+        gf_dirent_t      *tmp = NULL;
+        int              ret = 0;
+        loc_t            entry_loc = {0};
+        fd_t             *fd = NULL;
+        struct iatt      iattr = {0};
+
+        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
+                if (!_crawl_proceed (this, crawl_data->child,
+                                     crawl_data->crawl_flags, NULL)) {
+                        ret = -1;
+                        goto out;
+                }
+                *offset = entry->d_off;
+                if (IS_ENTRY_CWD (entry->d_name) ||
+                    IS_ENTRY_PARENT (entry->d_name))
+                        continue;
+                if ((crawl_data->crawl == FULL) &&
+                     uuid_is_null (entry->d_stat.ia_gfid)) {
+                        gf_log (this->name, GF_LOG_WARNING, "%s/%s: No "
+                                "gfid present skipping",
+                                parentloc->path, entry->d_name);
+                        continue;
+                }
+
+                loc_wipe (&entry_loc);
+                ret = afr_crawl_build_child_loc (this, &entry_loc, parentloc,
+                                                 entry, crawl_data);
+                if (ret)
+                        goto out;
+
+                ret = crawl_data->process_entry (this, crawl_data, entry,
+                                                 &entry_loc, parentloc, &iattr);
+
+                if (crawl_data->crawl == INDEX_TO_BE_HEALED && ret) {
+                       goto out;
+                } else if (ret) {
+                        continue;
+                }
+
+                if ((crawl_data->crawl == INDEX) ||
+                    (crawl_data->crawl == INDEX_TO_BE_HEALED))
+                        continue;
+
+                if (!IA_ISDIR (iattr.ia_type))
+                        continue;
+                fd = NULL;
+                ret = afr_crawl_opendir (this, crawl_data, &fd, &entry_loc);
+                if (ret)
+                        continue;
+                ret = _crawl_directory (fd, &entry_loc, crawl_data);
+                if (fd)
+                        fd_unref (fd);
+        }
+        ret = 0;
+out:
+        if ((crawl_data->crawl == INDEX_TO_BE_HEALED)  && ret) {
+                gf_log (this->name, GF_LOG_ERROR,"Failed to get the hardlink "
+                        "count");
+        }
+        loc_wipe (&entry_loc);
+        return ret;
+}
+
+static int
+_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data)
+{
+        xlator_t        *this = NULL;
+        off_t           offset   = 0;
+        gf_dirent_t     entries;
+        int             ret = 0;
+        gf_boolean_t    free_entries = _gf_false;
+        xlator_t        *readdir_xl = crawl_data->readdir_xl;
+
+        INIT_LIST_HEAD (&entries.list);
+        this = THIS;
+
+        GF_ASSERT (loc->inode);
+
+        if (crawl_data->crawl == FULL)
+                gf_log (this->name, GF_LOG_DEBUG, "crawling %s", loc->path);
+        else
+                gf_log (this->name, GF_LOG_DEBUG, "crawling INDEX %s",
+                        uuid_utoa (loc->gfid));
+
+        while (1) {
+                if (crawl_data->crawl == FULL)
+                        ret = syncop_readdirp (readdir_xl, fd, 131072, offset,
+                                               NULL, &entries);
+                else
+                        ret = syncop_readdir (readdir_xl, fd, 131072, offset,
+                                              &entries);
+                if (ret <= 0)
+                        break;
+                ret = 0;
+                free_entries = _gf_true;
+
+                if (!_crawl_proceed (this, crawl_data->child,
+                                     crawl_data->crawl_flags, NULL)) {
+                        ret = -1;
+                        goto out;
+                }
+                if (list_empty (&entries.list))
+                        goto out;
+
+                ret = _process_entries (this, loc, &entries, &offset,
+                                        crawl_data);
+                if ((ret < 0) && (crawl_data->crawl == INDEX_TO_BE_HEALED)) {
+                        goto out;
+                }
+                gf_dirent_free (&entries);
+                free_entries = _gf_false;
+        }
+        ret = 0;
+out:
+        if (free_entries)
+                gf_dirent_free (&entries);
+        return ret;
+}
+
+static char*
+position_str_get (afr_child_pos_t pos)
+{
+        switch (pos) {
+        case AFR_POS_UNKNOWN:
+                return "unknown";
+        case AFR_POS_LOCAL:
+                return "local";
+        case AFR_POS_REMOTE:
+                return "remote";
+        }
+        return NULL;
+}
+
+int
+afr_find_child_position (xlator_t *this, int child, afr_child_pos_t *pos)
+{
+        afr_private_t    *priv = NULL;
+        afr_self_heald_t *shd  = NULL;
+        dict_t           *xattr_rsp = NULL;
+        loc_t            loc = {0};
+        int              ret = 0;
+        char             *node_uuid = NULL;
+
+        priv = this->private;
+        shd  = &priv->shd;
+
+        afr_build_root_loc (this, &loc);
+
+        ret = syncop_getxattr (priv->children[child], &loc, &xattr_rsp,
+                               GF_XATTR_NODE_UUID_KEY);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "getxattr failed on %s - "
+                        "(%s)", priv->children[child]->name, strerror (errno));
+                goto out;
+        }
+
+        ret = dict_get_str (xattr_rsp, GF_XATTR_NODE_UUID_KEY, &node_uuid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "node-uuid key not found on "
+                        "child %s", priv->children[child]->name);
+                goto out;
+        }
+
+        if (!strcmp (node_uuid, shd->node_uuid))
+                *pos = AFR_POS_LOCAL;
+        else
+                *pos = AFR_POS_REMOTE;
+
+        gf_log (this->name, GF_LOG_DEBUG, "child %s is %s",
+                priv->children[child]->name, position_str_get (*pos));
+out:
+        if (ret)
+                *pos = AFR_POS_UNKNOWN;
+        loc_wipe (&loc);
+        return ret;
+}
+
+int
+afr_syncop_find_child_position (void *data)
+{
+        shd_pos_t *pos_data = data;
+        int       ret = 0;
+
+        ret = afr_find_child_position (pos_data->this, pos_data->child,
+                                       &pos_data->pos);
+        return ret;
+}
+
+static int
+afr_dir_crawl (void *data)
+{
+        xlator_t            *this = NULL;
+        int                 ret = -1;
+        xlator_t            *readdir_xl = NULL;
+        fd_t                *fd = NULL;
+        loc_t               dirloc = {0};
+        afr_crawl_data_t    *crawl_data = data;
+
+        this = THIS;
+
+        if (!_crawl_proceed (this, crawl_data->child, crawl_data->crawl_flags,
+                             NULL))
+                goto out;
+
+        readdir_xl = afr_crawl_readdir_xl_get (this, crawl_data);
+        if (!readdir_xl)
+                goto out;
+        crawl_data->readdir_xl = readdir_xl;
+
+        ret = afr_crawl_build_start_loc (this, crawl_data, &dirloc);
+        if (ret)
+                goto out;
+
+        ret = afr_crawl_opendir (this, crawl_data, &fd, &dirloc);
+        if (ret) {
+                if (crawl_data->crawl == INDEX_TO_BE_HEALED) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to open base_"
+                                "indices_holder");
+                }
+                goto out;
+        }
+
+        ret = _crawl_directory (fd, &dirloc, crawl_data);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Crawl failed on %s",
+                        readdir_xl->name);
+        else
+                gf_log (this->name, GF_LOG_DEBUG, "Crawl completed "
+                        "on %s", readdir_xl->name);
+        if (crawl_data->crawl == INDEX)
+                dirloc.path = NULL;
+out:
+        if (fd)
+                fd_unref (fd);
+        if ((crawl_data->crawl == INDEX) ||
+            (crawl_data->crawl == INDEX_TO_BE_HEALED ))
+                dirloc.path = NULL;
+        loc_wipe (&dirloc);
+        return ret;
+}
+
+char *
+get_crawl_type_in_string (afr_crawl_type_t crawl)
+{
+        char    *index = "INDEX";
+        char    *full  = "FULL";
+        char    *crawl_type = NULL;
+
+        if (crawl == INDEX){
+                crawl_type = index;
+        } else if (crawl == FULL) {
+                crawl_type = full;
+        }
+
+        return  crawl_type;
+}
+
+static int
+afr_allocate_crawl_event (xlator_t *this, int child, afr_crawl_type_t crawl)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     ret = 0;
+        shd_crawl_event_t       *crawl_event = NULL;
+        time_t                  get_time = 0;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        crawl_event = GF_CALLOC (sizeof (shd_crawl_event_t), 1,
+                                 gf_afr_mt_shd_crawl_event_t);
+        if (!crawl_event) {
+                ret = -1;
+                goto out;
+        }
+
+        get_time =  time(NULL);
+        if (get_time == ((time_t)-1)) {
+                 ret = -1;
+                goto out;
+        }
+
+        crawl_event->start_time_str = gf_strdup (ctime(&get_time));
+
+        crawl_event->crawl_type = get_crawl_type_in_string (crawl);
+        if (!crawl_event->crawl_type) {
+                ret = -1;
+                goto out;
+        }
+        LOCK (&priv->lock);
+        {
+                shd->crawl_events[child] = crawl_event;
+        }
+        UNLOCK (&priv->lock);
+        ret = 0;
+out:
+        return ret;
+
+}
+
+static int
+afr_put_crawl_event_in_eh (xlator_t *this, int child)
+{
+        afr_private_t           *priv = NULL;
+        afr_self_heald_t        *shd = NULL;
+        int                     ret = 0;
+        time_t                  get_time = 0;
+        shd_crawl_event_t       **crawl_event = NULL;
+
+        priv = this->private;
+        shd = &priv->shd;
+
+        get_time = time(NULL);
+        if (get_time == ((time_t)-1)) {
+                ret = -1;
+                goto out;
+        }
+        crawl_event = (shd_crawl_event_t**)shd->crawl_events;
+        LOCK (&priv->lock);
+        {
+                crawl_event[child]->end_time_str = gf_strdup (ctime(&get_time));
+                ret = eh_save_history (shd->statistics[child],
+                                       crawl_event[child]);
+                crawl_event[child] = NULL;
+        }
+        UNLOCK (&priv->lock);
+out:
+        return ret;
+}
+
+static int
+afr_dir_exclusive_crawl (void *data)
+{
+        afr_private_t    *priv = NULL;
+        afr_self_heald_t *shd = NULL;
+        gf_boolean_t     crawl = _gf_false;
+        int              ret = 0;
+        int              child = -1;
+        xlator_t         *this = NULL;
+        afr_crawl_data_t *crawl_data = data;
+
+        this = THIS;
+        priv = this->private;
+        shd = &priv->shd;
+        child = crawl_data->child;
+
+        LOCK (&priv->lock);
+        {
+                if (shd->inprogress[child]) {
+                        if (shd->pending[child] != FULL)
+                                shd->pending[child] = crawl_data->crawl;
+                } else {
+                        shd->inprogress[child] = _gf_true;
+                        crawl = _gf_true;
+                }
+        }
+        UNLOCK (&priv->lock);
+
+        if (!crawl) {
+                gf_log (this->name, GF_LOG_INFO, "Another crawl is in progress "
+                        "for %s", priv->children[child]->name);
+                goto out;
+        }
+
+        do {
+                ret = afr_allocate_crawl_event (this, child, crawl_data->crawl);
+                if (ret)
+                        goto out;
+                afr_dir_crawl (data);
+
+                ret = afr_put_crawl_event_in_eh (this, child);
+                if (ret < 0)
+                        goto out;
+
+                LOCK (&priv->lock);
+                {
+                        if (shd->pending[child] != NONE) {
+                                crawl_data->crawl = shd->pending[child];
+                                shd->pending[child] = NONE;
+                        } else {
+                                shd->inprogress[child] = _gf_false;
+                                crawl = _gf_false;
+                        }
+                }
+                UNLOCK (&priv->lock);
+        } while (crawl);
+out:
+        return ret;
+}
+
+void
+afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
+                 process_entry_cbk_t process_entry, void *op_data,
+                 gf_boolean_t exclusive, int crawl_flags,
+                 afr_crawl_done_cbk_t crawl_done)
+{
+        afr_private_t              *priv = NULL;
+        call_frame_t               *frame = NULL;
+        afr_crawl_data_t           *crawl_data = NULL;
+        int                        ret = 0;
+        int (*crawler) (void*) = NULL;
+
+        priv = this->private;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        afr_set_lk_owner (frame, this, frame->root);
+        afr_set_low_priority (frame);
+        crawl_data = GF_CALLOC (1, sizeof (*crawl_data),
+                                gf_afr_mt_crawl_data_t);
+        if (!crawl_data)
+                goto out;
+        crawl_data->process_entry = process_entry;
+        crawl_data->child = idx;
+        crawl_data->pid = frame->root->pid;
+        crawl_data->crawl = crawl;
+        crawl_data->op_data = op_data;
+        crawl_data->crawl_flags = crawl_flags;
+        gf_log (this->name, GF_LOG_DEBUG, "starting crawl %d for %s",
+                crawl_data->crawl, priv->children[idx]->name);
+
+        if (exclusive)
+                crawler = afr_dir_exclusive_crawl;
+        else
+                crawler = afr_dir_crawl;
+        ret = synctask_new (this->ctx->env, crawler,
+                            crawl_done, frame, crawl_data);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "afr crawl failed for child"
+                        " %d with ret %d", idx, ret);
+out:
+        return;
+}
+
+void
+afr_build_root_loc (xlator_t *this, loc_t *loc)
+{
+        afr_private_t   *priv = NULL;
+
+        priv = this->private;
+        loc->path = gf_strdup ("/");
+        loc->name = "";
+        loc->inode = inode_ref (priv->root_inode);
+        uuid_copy (loc->gfid, loc->inode->gfid);
+}
+
+int
+afr_set_root_gfid (dict_t *dict)
+{
+        uuid_t gfid;
+        int ret = 0;
+
+        memset (gfid, 0, 16);
+        gfid[15] = 1;
+
+        ret = afr_set_dict_gfid (dict, gfid);
+
+        return ret;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
new file mode 100644
index 000000000..e0c083754
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -0,0 +1,65 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __AFR_SELF_HEALD_H__
+#define __AFR_SELF_HEALD_H__
+#include "xlator.h"
+
+#define IS_ROOT_PATH(path) (!strcmp (path, "/"))
+#define IS_ENTRY_CWD(entry) (!strcmp (entry, "."))
+#define IS_ENTRY_PARENT(entry) (!strcmp (entry, ".."))
+#define AFR_ALL_CHILDREN -1
+
+typedef struct afr_crawl_data_ {
+        int                 child;
+        pid_t               pid;
+        afr_crawl_type_t    crawl;
+        xlator_t            *readdir_xl;
+        void                *op_data;
+        int                 crawl_flags;
+        int (*process_entry) (xlator_t *this, struct afr_crawl_data_ *crawl_data,
+                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
+                              struct iatt *iattr);
+} afr_crawl_data_t;
+
+typedef struct crawl_event_stats_ {
+        uint64_t healed_count;
+        uint64_t split_brain_count;
+        uint64_t heal_failed_count;
+        char     *start_time_str;
+        char     *end_time_str;
+        char     *crawl_type;
+        gf_boolean_t crawl_inprogress;
+} shd_crawl_event_t;
+
+void _destroy_crawl_event_data (void *data);
+void _destroy_shd_event_data (void *data);
+
+typedef int (*process_entry_cbk_t) (xlator_t *this, afr_crawl_data_t *crawl_data,
+                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
+                              struct iatt *iattr);
+
+void afr_build_root_loc (xlator_t *this, loc_t *loc);
+
+int afr_set_root_gfid (dict_t *dict);
+
+void
+afr_proactive_self_heal (void *data);
+
+int
+afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+
+/*
+ * In addition to its self-heal use, this is used to find a local default
+ * read_child.
+ */
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *local);
+#endif /* __AFR_SELF_HEALD_H__ */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index f9a0a8806..20306e469 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -1,24 +1,17 @@
 /*
-  Copyright (c) 2007, 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "dict.h"
 #include "byte-order.h"
+#include "common-utils.h"
+#include "timer.h"
 
 #include "afr.h"
 #include "afr-transaction.h"
@@ -26,936 +19,1945 @@
 #include <signal.h>
 
 
-static void
-__mark_all_pending (int32_t *pending, int child_count)
-{	
-	int i;
-	
-	for (i = 0; i < child_count; i++)
-		pending[i] = hton32 (1);
+#define LOCKED_NO       0x0        /* no lock held */
+#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path
+                                      of RENAME */
+#define LOCKED_LOWER    0x2        /* for lower_path of RENAME */
+
+afr_fd_ctx_t *
+__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+{
+        uint64_t       ctx = 0;
+        int            ret = 0;
+        afr_fd_ctx_t  *fd_ctx = NULL;
+        int            i = 0;
+        afr_private_t *priv = NULL;
+
+        priv = this->private;
+
+        ret = __fd_ctx_get (fd, this, &ctx);
+
+        if (ret < 0 && fd_is_anonymous (fd)) {
+                ret = __afr_fd_ctx_set (this, fd);
+                if (ret < 0)
+                        goto out;
+
+                ret = __fd_ctx_get (fd, this, &ctx);
+                if (ret < 0)
+                        goto out;
+
+                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+                for (i = 0; i < priv->child_count; i++)
+                        fd_ctx->opened_on[i] = AFR_FD_OPENED;
+        }
+
+        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+out:
+        return fd_ctx;
+}
+
+
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+{
+        afr_fd_ctx_t  *fd_ctx = NULL;
+
+        LOCK(&fd->lock);
+        {
+                fd_ctx = __afr_fd_ctx_get (fd, this);
+        }
+        UNLOCK(&fd->lock);
+
+        return fd_ctx;
 }
 
 
 static void
-__mark_child_dead (int32_t *pending, int child_count, int child)
+afr_save_lk_owner (call_frame_t *frame)
 {
-	pending[child] = 0;
+        afr_local_t * local = NULL;
+
+        local = frame->local;
+
+        local->saved_lk_owner = frame->root->lk_owner;
 }
 
 
 static void
-__mark_down_children (int32_t *pending, int child_count, unsigned char *child_up)
+afr_restore_lk_owner (call_frame_t *frame)
 {
-	int i;
-	
-	for (i = 0; i < child_count; i++)
-		if (!child_up[i])
-			pending[i] = 0;
+        afr_local_t * local = NULL;
+
+        local = frame->local;
+
+        frame->root->lk_owner = local->saved_lk_owner;
+}
+
+static void
+__mark_all_pending (int32_t *pending[], int child_count,
+                    afr_transaction_type type)
+{
+        int i = 0;
+        int j = 0;
+
+        for (i = 0; i < child_count; i++) {
+                j = afr_index_for_transaction_type (type);
+                pending[i][j] = hton32 (1);
+        }
 }
 
 
 static void
-__mark_all_success (int32_t *pending, int child_count)
+__mark_child_dead (int32_t *pending[], int child_count, int child,
+                   afr_transaction_type type)
 {
-	int i;
-	
-	for (i = 0; i < child_count; i++)
-		pending[i] = hton32 (-1);
+        int j = 0;
+
+        j = afr_index_for_transaction_type (type);
+
+        pending[child][j] = 0;
 }
 
 
-static int
-__is_first_write_on_fd (xlator_t *this, fd_t *fd)
+static void
+__mark_pre_op_done_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
 {
-        int op_ret     = 0;
-        int _ret       = -1;
+        afr_local_t   *local = NULL;
+        afr_fd_ctx_t  *fd_ctx = NULL;
 
-        LOCK (&fd->inode->lock);
+        local = frame->local;
+
+        if (!local->fd)
+                return;
+
+        fd_ctx = afr_fd_ctx_get (local->fd, this);
+
+        if (!fd_ctx)
+                goto out;
+
+        LOCK (&local->fd->lock);
         {
-                _ret = fd_ctx_get (fd, this, NULL);
-                if (_ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "first writev() on fd=%p, writing changelog",
-                                fd);
+                if (local->transaction.type == AFR_DATA_TRANSACTION)
+                        fd_ctx->pre_op_done[child_index]++;
+        }
+        UNLOCK (&local->fd->lock);
+out:
+        return;
+}
 
-                        _ret = fd_ctx_set (fd, this, 0xaf1);
-                        op_ret = 1;
-                }
+static void
+__mark_non_participant_children (int32_t *pending[], int child_count,
+                                 unsigned char *participants,
+                                 afr_transaction_type type)
+{
+        int i = 0;
+        int j = 0;
+
+        j = afr_index_for_transaction_type (type);
+        for (i = 0; i < child_count; i++) {
+                if (!participants[i])
+                        pending[i][j] = 0;
         }
-        UNLOCK (&fd->inode->lock);
+}
 
-        return op_ret;
+
+void
+__mark_all_success (int32_t *pending[], int child_count,
+                    afr_transaction_type type)
+{
+        int i;
+        int j;
+
+        for (i = 0; i < child_count; i++) {
+                j = afr_index_for_transaction_type (type);
+                pending[i][j] = hton32 (-1);
+        }
+}
+
+void
+_set_all_child_errno (int *child_errno, unsigned int child_count)
+{
+        int     i = 0;
+
+        for (i = 0; i < child_count; i++)
+                if (child_errno[i] == 0)
+                        child_errno[i] = ENOTCONN;
+}
+
+void
+afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
+        fd_t            *fd   = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+        fd    = local->fd;
+
+        __mark_all_success (local->pending, priv->child_count,
+                            local->transaction.type);
+
+        _set_all_child_errno (local->child_errno, priv->child_count);
+
+        /*  Perform fops with the lk-owner from top xlator.
+         *  Eg: lk-owner of posix-lk and flush should be same,
+         *  flush cant clear the  posix-lks without that lk-owner.
+         */
+        afr_save_lk_owner (frame);
+        frame->root->lk_owner =
+                local->transaction.main_frame->root->lk_owner;
+
+
+        /* The wake up needs to happen independent of
+           what type of fop arrives here. If it was
+           a write, then it has already inherited the
+           lock and changelog. If it was not a write,
+           then the presumption of the optimization (of
+           optimizing for successive write operations)
+           fails.
+        */
+        if (fd)
+                afr_delayed_changelog_wake_up (this, fd);
+        local->transaction.fop (frame, this);
 }
 
 
 static int
 __changelog_enabled (afr_private_t *priv, afr_transaction_type type)
 {
-	int ret = 0;
+        int ret = 0;
 
-	switch (type) {
-	case AFR_DATA_TRANSACTION:
-		if (priv->data_change_log)
-			ret = 1;
-		
-		break;
+        switch (type) {
+        case AFR_DATA_TRANSACTION:
+                if (priv->data_change_log)
+                        ret = 1;
 
-	case AFR_METADATA_TRANSACTION:
-		if (priv->metadata_change_log)
-			ret = 1;
+                break;
 
-		break;
+        case AFR_METADATA_TRANSACTION:
+                if (priv->metadata_change_log)
+                        ret = 1;
 
-	case AFR_ENTRY_TRANSACTION:
-	case AFR_ENTRY_RENAME_TRANSACTION:
-		if (priv->entry_change_log)
-			ret = 1;
+                break;
 
-		break;
-		
-	case AFR_FLUSH_TRANSACTION:
-		ret = 1;
-	}
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+                if (priv->entry_change_log)
+                        ret = 1;
+
+                break;
+        }
 
-	return ret;
+        return ret;
 }
 
 
 static int
-__changelog_needed_pre_op (call_frame_t *frame, xlator_t *this)
+__fop_changelog_needed (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	fd_t *          fd    = NULL;
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        int op_ret = 0;
+        afr_transaction_type type = -1;
 
-	int op_ret   = 0;
+        priv  = this->private;
+        local = frame->local;
+        type  = local->transaction.type;
 
-	priv  = this->private;
-	local = frame->local;
-	
-	if (__changelog_enabled (priv, local->transaction.type)) {
-		switch (local->op) {
-
-		case GF_FOP_WRITE:
-		case GF_FOP_FTRUNCATE:
-			/* 
-			   if it's a data transaction, we write the changelog
-			   only on the first write on an fd 
-			*/
-			
-			fd = local->fd;
-			if (!fd || __is_first_write_on_fd (this, fd))
-				op_ret = 1;
-
-			break;
-
-		case GF_FOP_FLUSH:
-			/* only do post-op on flush() */
-
-			op_ret = 0;
-			break;
-
-		default:
-			op_ret = 1;
-		}
-	}
+        if (__changelog_enabled (priv, type)) {
+                switch (local->op) {
 
-	return op_ret;
+                case GF_FOP_WRITE:
+                case GF_FOP_FTRUNCATE:
+                        op_ret = 1;
+                        break;
+
+                case GF_FOP_FLUSH:
+                        op_ret = 0;
+                        break;
+
+                default:
+                        op_ret = 1;
+                }
+        }
+
+        return op_ret;
 }
 
+int
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
+                      int child, afr_xattrop_type_t op)
+{
+        int i = 0;
+        int ret = 0;
+
+        if (op == LOCAL_FIRST) {
+                ret = dict_set_static_bin (xattr, priv->pending_key[child],
+                                           pending[child],
+                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                if (ret)
+                        goto out;
+        }
+        for (i = 0; i < priv->child_count; i++) {
+                if (i == child)
+                        continue;
+                ret = dict_set_static_bin (xattr, priv->pending_key[i],
+                                           pending[i],
+                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                /* 3 = data+metadata+entry */
+
+                if (ret < 0)
+                        goto out;
+        }
+        if (op == LOCAL_LAST) {
+                ret = dict_set_static_bin (xattr, priv->pending_key[child],
+                                           pending[child],
+                                   AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+                if (ret)
+                        goto out;
+        }
+out:
+        return ret;
+}
 
-static int
-__changelog_needed_post_op (call_frame_t *frame, xlator_t *this)
+int
+afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
+        int ret = 0;
 
-	int ret = 0;
-	afr_transaction_type type = -1;
+        switch (type) {
+        case AFR_DATA_TRANSACTION:
+                ret = priv->child_count;
+                break;
 
-	priv  = this->private;
-	local = frame->local;
-	type  = local->transaction.type;
+        case AFR_METADATA_TRANSACTION:
+                ret = priv->child_count;
+                break;
 
-	if (__changelog_enabled (priv, type)
-	    && (local->op != GF_FOP_WRITE)
-	    && (local->op != GF_FOP_FTRUNCATE))
-		ret = 1;
-	
-	return ret;
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+                ret = priv->child_count;
+                break;
+        }
+
+        return ret;
 }
 
+/* {{{ pending */
 
-static int
-afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
+int32_t
+afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
-	int ret = 0;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_private_t       *priv     = NULL;
+        afr_local_t         *local    = NULL;
+        int                  call_count = -1;
+
+        priv     = this->private;
+        local    = frame->local;
+        int_lock = &local->internal_lock;
 
-	switch (type) {
-	case AFR_FLUSH_TRANSACTION:
-	case AFR_DATA_TRANSACTION:
-		ret = priv->data_lock_server_count;
-		break;
+        LOCK (&frame->lock);
+        {
+                call_count = --local->call_count;
+        }
+        UNLOCK (&frame->lock);
 
-	case AFR_METADATA_TRANSACTION:
-		ret = priv->metadata_lock_server_count;
-		break;
+        if (call_count == 0) {
+                if (local->transaction.resume_stub) {
+			call_resume (local->transaction.resume_stub);
+                        local->transaction.resume_stub = NULL;
+                }
 
-	case AFR_ENTRY_TRANSACTION:
-	case AFR_ENTRY_RENAME_TRANSACTION:
-		ret = priv->entry_lock_server_count;
-		break;
-	}
+                if (afr_lock_server_count (priv, local->transaction.type) == 0) {
+                        local->transaction.done (frame, this);
+                } else {
+                        int_lock->lock_cbk = local->transaction.done;
+                        afr_unlock (frame, this);
+                }
+        }
 
-	return ret;
+        return 0;
 }
 
 
-/* {{{ unlock */
-
-int32_t
-afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+void
+afr_transaction_rm_stale_children (call_frame_t *frame, xlator_t *this,
+                                   inode_t *inode, afr_transaction_type type)
 {
-	afr_local_t *local;
-	int call_count = 0;
+        int             i = -1;
+        int             count = 0;
+        int             read_child = -1;
+        afr_private_t   *priv = NULL;
+        afr_local_t     *local = NULL;
+        int             **pending = NULL;
+        int             idx = 0;
+        int32_t         *stale_children = NULL;
+        int32_t         *fresh_children = NULL;
+        gf_boolean_t    rm_stale_children = _gf_false;
+
+        idx = afr_index_for_transaction_type (type);
+
+        priv = this->private;
+        local = frame->local;
+        pending = local->pending;
+
+        if (local->op_ret < 0)
+                goto out;
+        fresh_children = local->fresh_children;
+        read_child = afr_inode_get_read_ctx (this, inode, fresh_children);
+        if (read_child < 0) {
+                gf_log (this->name, GF_LOG_DEBUG, "Possible split-brain "
+                        "for %s", uuid_utoa (inode->gfid));
+                goto out;
+        }
 
-	local = frame->local;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!afr_is_child_present (fresh_children,
+                                           priv->child_count, i))
+                        continue;
+                if (pending[i][idx])
+                        continue;
+                /* child is down or op failed on it */
+                if (!stale_children)
+                        stale_children = afr_children_create (priv->child_count);
+                if (!stale_children)
+                        goto out;
+
+                rm_stale_children = _gf_true;
+                stale_children[count++] = i;
+                gf_log (this->name, GF_LOG_DEBUG, "Removing stale child "
+                        "%d for %s", i, uuid_utoa (inode->gfid));
+        }
 
-	LOCK (&frame->lock);
-	{
-		call_count = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
+        if (!rm_stale_children)
+                goto out;
 
-	if (call_count == 0) {
-		local->transaction.done (frame, this);
-	}
-	
-	return 0;
+        afr_inode_rm_stale_children (this, inode, stale_children);
+out:
+        GF_FREE (stale_children);
+        return;
 }
 
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom)
+{
+        afr_inodelk_t *inodelk = NULL;
+        int           i = 0;
+
+        for (i = 0; int_lock->inodelk[i].domain; i++) {
+                inodelk = &int_lock->inodelk[i];
+                if (strcmp (dom, inodelk->domain) == 0)
+                        return inodelk;
+        }
+        return NULL;
+}
+
+unsigned char*
+afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
+{
+        unsigned char *locked_nodes = NULL;
+        afr_inodelk_t *inodelk = NULL;
+        switch (type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                locked_nodes = inodelk->locked_nodes;
+        break;
+
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+                /*Because same set of subvols participate in all lockee
+                 * entities*/
+                locked_nodes = int_lock->lockee[0].locked_nodes;
+        break;
+        }
+        return locked_nodes;
+}
 
 int
-afr_unlock (call_frame_t *frame, xlator_t *this)
+afr_changelog_pre_op_call_count (afr_transaction_type type,
+                                 afr_internal_lock_t *int_lock,
+                                 unsigned int child_count)
 {
-	struct flock flock;			
+        int           call_count = 0;
+        unsigned char *locked_nodes = NULL;
 
-	int i = 0;				
-	int call_count = 0;		     
+        locked_nodes = afr_locked_nodes_get (type, int_lock);
+        GF_ASSERT (locked_nodes);
 
-	afr_local_t *local = NULL;
-	afr_private_t * priv = this->private;
+        call_count = afr_locked_children_count (locked_nodes, child_count);
+        if (type == AFR_ENTRY_RENAME_TRANSACTION)
+                call_count *= 2;
 
-	local = frame->local;
-	
-	call_count = afr_locked_nodes_count (local->transaction.locked_nodes, 
-					     priv->child_count);
-	
-	if (call_count == 0) {
-		local->transaction.done (frame, this);
-		return 0;
-	}
+        return call_count;
+}
 
-	if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) 
-		call_count *= 2;
-
-	local->call_count = call_count;		
-
-	for (i = 0; i < priv->child_count; i++) {				
-		flock.l_start = local->transaction.start;			
-		flock.l_len   = local->transaction.len;
-		flock.l_type  = F_UNLCK;			
-
-		if (local->transaction.locked_nodes[i]) {
-			switch (local->transaction.type) {
-			case AFR_DATA_TRANSACTION:
-			case AFR_METADATA_TRANSACTION:
-			case AFR_FLUSH_TRANSACTION:
-
-				if (local->fd) {
-					STACK_WIND (frame, afr_unlock_common_cbk,	
-						    priv->children[i], 
-						    priv->children[i]->fops->finodelk, 
-						    local->fd, F_SETLK, &flock); 
-				} else {
-					STACK_WIND (frame, afr_unlock_common_cbk,	
-						    priv->children[i], 
-						    priv->children[i]->fops->inodelk, 
-						    &local->loc,  F_SETLK, &flock); 
-				}
-				
-				break;
-
-			case AFR_ENTRY_RENAME_TRANSACTION:
-				
-				STACK_WIND (frame, afr_unlock_common_cbk,	
-					    priv->children[i], 
-					    priv->children[i]->fops->entrylk, 
-					    &local->transaction.new_parent_loc, 
-					    local->transaction.new_basename,
-					    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-
-				call_count--;
-
-				/* fall through */
-
-			case AFR_ENTRY_TRANSACTION:
-				if (local->fd) {
-					STACK_WIND (frame, afr_unlock_common_cbk,	
-						    priv->children[i], 
-						    priv->children[i]->fops->fentrylk, 
-						    local->fd, 
-						    local->transaction.basename,
-						    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-				} else {
-					STACK_WIND (frame, afr_unlock_common_cbk,	
-						    priv->children[i], 
-						    priv->children[i]->fops->entrylk, 
-						    &local->transaction.parent_loc, 
-						    local->transaction.basename,
-						    ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-
-				}
-				break;
-			}
-			
-			if (!--call_count)
-				break;
-		}
-	}
+int
+afr_changelog_post_op_call_count (afr_transaction_type type,
+                                  unsigned char *pre_op,
+                                  unsigned int child_count)
+{
+        int           call_count = 0;
+
+        call_count = afr_pre_op_done_children_count (pre_op, child_count);
+        if (type == AFR_ENTRY_RENAME_TRANSACTION)
+                call_count *= 2;
 
-	return 0;
+        return call_count;
 }
 
-/* }}} */
+void
+afr_compute_txn_changelog (afr_local_t *local, afr_private_t *priv)
+{
+        int             i = 0;
+        int             index = 0;
+        int32_t         postop = 0;
+        int32_t         preop = 1;
+        int32_t         **txn_changelog = NULL;
+
+        txn_changelog = local->transaction.txn_changelog;
+        index = afr_index_for_transaction_type (local->transaction.type);
+        for (i = 0; i < priv->child_count; i++) {
+                postop = ntoh32 (local->pending[i][index]);
+                txn_changelog[i][index] = hton32 (postop + preop);
+        }
+}
 
+afr_xattrop_type_t
+afr_get_postop_xattrop_type (int32_t **pending, int optimized, int child,
+                             afr_transaction_type type)
+{
+        int                     index = 0;
+        afr_xattrop_type_t      op = LOCAL_LAST;
 
-/* {{{ pending */
+        index = afr_index_for_transaction_type (type);
+        if (optimized && !pending[child][index])
+                op = LOCAL_FIRST;
+        return op;
+}
 
-int32_t
-afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno, dict_t *xattr)
+void
+afr_set_postop_dict (afr_local_t *local, xlator_t *this, dict_t *xattr,
+                     int optimized, int child)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t *   local = NULL;
-	
-	int call_count = -1;
+        int32_t                 **txn_changelog = NULL;
+        int32_t                 **changelog = NULL;
+        afr_private_t           *priv = NULL;
+        int                     ret = 0;
+        afr_xattrop_type_t      op = LOCAL_LAST;
+
+        priv = this->private;
+        txn_changelog = local->transaction.txn_changelog;
+        op = afr_get_postop_xattrop_type (local->pending, optimized, child,
+                                          local->transaction.type);
+        if (optimized)
+                changelog = txn_changelog;
+        else
+                changelog = local->pending;
+        ret = afr_set_pending_dict (priv, xattr, changelog, child, op);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_INFO,
+                        "failed to set pending entry");
+}
 
-	priv  = this->private;
-	local = frame->local;
 
-	LOCK (&frame->lock);
-	{
-		call_count = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (call_count == 0) {
-		if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-			local->transaction.done (frame, this);
-		} else {
-			afr_unlock (frame, this);
-		}
-	}
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int index = -1;
+        int i = 0;
+
+        local = frame->local;
+        priv = this->private;
 
-	return 0;	
+        index = afr_index_for_transaction_type (local->transaction.type);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->pending[i][index] == 0)
+                        return _gf_false;
+        }
+
+        return _gf_true;
 }
 
+static void
+afr_dir_fop_handle_all_fop_failures (call_frame_t *frame)
+{
+        xlator_t        *this = NULL;
+        afr_local_t     *local = NULL;
+        afr_private_t   *priv = NULL;
 
-int 
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+        this = frame->this;
+        local = frame->local;
+        priv = this->private;
+
+        if ((local->transaction.type != AFR_ENTRY_TRANSACTION) &&
+            (local->transaction.type != AFR_ENTRY_RENAME_TRANSACTION))
+                return;
+
+        if (local->op_ret >= 0)
+                goto out;
+
+        __mark_all_success (local->pending, priv->child_count,
+                            local->transaction.type);
+out:
+        return;
+}
+
+static void
+afr_data_handle_quota_errors (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv = this->private;
+        int     i = 0;
+        afr_private_t *priv = NULL;
+        afr_local_t   *local = NULL;
+        gf_boolean_t  all_quota_failures = _gf_false;
+
+        local = frame->local;
+        priv  = this->private;
+        if (local->transaction.type != AFR_DATA_TRANSACTION)
+                return;
+        /*
+         * Idea is to not leave the file in FOOL-FOOL scenario in case on
+         * all the bricks data transaction failed with EDQUOT to avoid
+         * increasing un-necessary load of self-heals in the system.
+         */
+        all_quota_failures = _gf_true;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i] &&
+                    (local->child_errno[i] != EDQUOT)) {
+                        all_quota_failures = _gf_false;
+                        break;
+                }
+        }
+        if (all_quota_failures)
+                __mark_all_success (local->pending, priv->child_count,
+                                    local->transaction.type);
+}
 
-	int ret        = 0;
-	int i          = 0;				
-	int call_count = 0;
-	
-	afr_local_t *  local = NULL;	
-	dict_t *       xattr = dict_ref (get_new_dict ());
+int
+afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t * priv = this->private;
+        afr_internal_lock_t *int_lock = NULL;
+        int i          = 0;
+        int call_count = 0;
+
+        afr_local_t *  local = NULL;
+        afr_fd_ctx_t  *fdctx = NULL;
+        dict_t        **xattr = NULL;
+        int            piggyback = 0;
+        int            nothing_failed = 1;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        __mark_non_participant_children (local->pending, priv->child_count,
+                                         local->transaction.pre_op,
+                                         local->transaction.type);
+
+        afr_data_handle_quota_errors (frame, this);
+        afr_dir_fop_handle_all_fop_failures (frame);
+
+        if (local->fd)
+                afr_transaction_rm_stale_children (frame, this,
+                                                   local->fd->inode,
+                                                   local->transaction.type);
+
+        xattr = alloca (priv->child_count * sizeof (*xattr));
+        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
+        for (i = 0; i < priv->child_count; i++) {
+                xattr[i] = dict_new ();
+        }
 
-	local = frame->local;
+        call_count = afr_changelog_post_op_call_count (local->transaction.type,
+                                                       local->transaction.pre_op,
+                                                       priv->child_count);
+        local->call_count = call_count;
 
-	__mark_all_success (local->pending_array, priv->child_count);
-	__mark_down_children (local->pending_array, priv->child_count, local->child_up);
+        if (local->fd)
+                fdctx = afr_fd_ctx_get (local->fd, this);
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up); 
+        if (call_count == 0) {
+                /* no child is up */
+                int_lock->lock_cbk = local->transaction.done;
+                afr_unlock (frame, this);
+                goto out;
+        }
 
-	if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-		call_count *= 2;
-	}
+        nothing_failed = afr_txn_nothing_failed (frame, this);
+
+        afr_compute_txn_changelog (local , priv);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
+                        continue;
+
+                if (local->transaction.type != AFR_DATA_TRANSACTION)
+                        afr_set_postop_dict (local, this, xattr[i],
+                                             local->optimistic_change_log, i);
+                switch (local->transaction.type) {
+                case AFR_DATA_TRANSACTION:
+                {
+                        if (!fdctx) {
+                                afr_set_postop_dict (local, this, xattr[i],
+                                                     0, i);
+                                STACK_WIND (frame, afr_changelog_post_op_cbk,
+                                            priv->children[i],
+                                            priv->children[i]->fops->xattrop,
+                                            &local->loc,
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
+                                break;
+                        }
+
+                        /* local->transaction.postop_piggybacked[] was
+                           precomputed in is_piggyback_postop() when called from
+                           afr_changelog_post_op_safe()
+                        */
+
+                        piggyback = 0;
+                        if (local->transaction.postop_piggybacked[i])
+                                piggyback = 1;
+
+                        afr_set_postop_dict (local, this, xattr[i],
+                                             piggyback, i);
+
+                        if (nothing_failed && piggyback) {
+                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
+                                                           this, 1, 0, xattr[i], NULL);
+                        } else {
+                                STACK_WIND_COOKIE (frame,
+                                                   afr_changelog_post_op_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[i],
+                                                   priv->children[i]->fops->fxattrop,
+                                                   local->fd,
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
+                        }
+                }
+                break;
+                case AFR_METADATA_TRANSACTION:
+                {
+                        if (nothing_failed && local->optimistic_change_log) {
+                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
+                                                           this, 1, 0, xattr[i],
+                                                           NULL);
+                                break;
+                        }
+
+                        if (local->fd)
+                                STACK_WIND (frame, afr_changelog_post_op_cbk,
+                                            priv->children[i],
+                                            priv->children[i]->fops->fxattrop,
+                                            local->fd,
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
+                        else
+                                STACK_WIND (frame, afr_changelog_post_op_cbk,
+                                            priv->children[i],
+                                            priv->children[i]->fops->xattrop,
+                                            &local->loc,
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
+                }
+                break;
+
+                case AFR_ENTRY_RENAME_TRANSACTION:
+                {
+                        if (nothing_failed && local->optimistic_change_log) {
+                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
+                                                           this, 1, 0, xattr[i],
+                                                           NULL);
+                        } else {
+                                STACK_WIND_COOKIE (frame, afr_changelog_post_op_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[i],
+                                                   priv->children[i]->fops->xattrop,
+                                                   &local->transaction.new_parent_loc,
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
+                        }
+                        call_count--;
+                }
 
-	local->call_count = call_count;		
+                /*
+                  set it again because previous stack_wind
+                  might have already returned (think of case
+                  where subvolume is posix) and would have
+                  used the dict as placeholder for return
+                  value
+                */
+
+                afr_set_postop_dict (local, this, xattr[i],
+                                     local->optimistic_change_log, i);
+
+                /* fall through */
+
+                case AFR_ENTRY_TRANSACTION:
+                {
+                        if (nothing_failed && local->optimistic_change_log) {
+                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
+                                                           this, 1, 0, xattr[i],
+                                                           NULL);
+                                break;
+                        }
+
+                        if (local->fd)
+                                STACK_WIND (frame, afr_changelog_post_op_cbk,
+                                            priv->children[i],
+                                            priv->children[i]->fops->fxattrop,
+                                            local->fd,
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
+                        else
+                                STACK_WIND (frame, afr_changelog_post_op_cbk,
+                                            priv->children[i],
+                                            priv->children[i]->fops->xattrop,
+                                            &local->transaction.parent_loc,
+                                            GF_XATTROP_ADD_ARRAY, xattr[i],
+                                            NULL);
+                }
+                break;
+                }
 
-	if (call_count == 0) {
-		/* no child is up */
-		dict_unref (xattr);
-		afr_unlock (frame, this);
-		return 0;
-	}
+                if (!--call_count)
+                        break;
+        }
 
-	for (i = 0; i < priv->child_count; i++) {					
-		if (local->child_up[i]) {
-			ret = dict_set_static_bin (xattr, local->transaction.pending, 
-						   local->pending_array, 
-						   priv->child_count * sizeof (int32_t));
-			if (ret < 0)
-				gf_log (this->name, GF_LOG_ERROR, 
-					"failed to set pending entry");
-
-
-			switch (local->transaction.type) {
-			case AFR_DATA_TRANSACTION:
-			case AFR_METADATA_TRANSACTION:
-			case AFR_FLUSH_TRANSACTION:
-			{
-				if (local->fd)
-					STACK_WIND (frame, afr_changelog_post_op_cbk,
-						    priv->children[i], 
-						    priv->children[i]->fops->fxattrop,
-						    local->fd, 
-						    GF_XATTROP_ADD_ARRAY, xattr);
-				else 
-					STACK_WIND (frame, afr_changelog_post_op_cbk,
-						    priv->children[i], 
-						    priv->children[i]->fops->xattrop,
-						    &local->loc, 
-						    GF_XATTROP_ADD_ARRAY, xattr);
-			}
-			break;
-
-			case AFR_ENTRY_RENAME_TRANSACTION:
-			{
-				STACK_WIND_COOKIE (frame, afr_changelog_post_op_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->xattrop,
-						   &local->transaction.new_parent_loc,
-						   GF_XATTROP_ADD_ARRAY, xattr);
-				
-				call_count--;
-			}
-
-			/* 
-			   set it again because previous stack_wind
-			   might have already returned (think of case
-			   where subvolume is posix) and would have
-			   used the dict as placeholder for return
-			   value
-			*/
-			ret = dict_set_static_bin (xattr, local->transaction.pending, 
-						   local->pending_array, 
-						   priv->child_count * sizeof (int32_t));
-			if (ret < 0)
-				gf_log (this->name, GF_LOG_ERROR, 
-					"failed to set pending entry");
-
-			/* fall through */
-
-			case AFR_ENTRY_TRANSACTION:
-			{
-				if (local->fd)
-					STACK_WIND (frame, afr_changelog_post_op_cbk,
-						    priv->children[i], 
-						    priv->children[i]->fops->fxattrop,
-						    local->fd, 
-						    GF_XATTROP_ADD_ARRAY, xattr);
-				else 
-					STACK_WIND (frame, afr_changelog_post_op_cbk,
-						    priv->children[i], 
-						    priv->children[i]->fops->xattrop,
-						    &local->transaction.parent_loc, 
-						    GF_XATTROP_ADD_ARRAY, xattr);
-			}
-			break;
-			}
-
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	dict_unref (xattr);
-	return 0;
+out:
+        for (i = 0; i < priv->child_count; i++) {
+                dict_unref (xattr[i]);
+        }
+
+        return 0;
 }
 
 
 int32_t
 afr_changelog_pre_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			      int32_t op_ret, int32_t op_errno, dict_t *xattr)
+                          int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                          dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = this->private;
-	loc_t       *   loc   = NULL;
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = this->private;
+        int call_count  = -1;
+        int child_index = (long) cookie;
 
-	int call_count  = -1;
-	int child_index = (long) cookie;
+        local = frame->local;
 
-	local = frame->local;
-	loc   = &local->loc;
+        LOCK (&frame->lock);
+        {
+                switch (op_ret) {
+                case 0:
+                        __mark_pre_op_done_on_fd (frame, this, child_index);
+                        //fallthrough we need to mark the pre_op
+                case 1:
+                        local->transaction.pre_op[child_index] = 1;
+                        /* special op_ret for piggyback */
+                        break;
+                case -1:
+                        if (op_errno == ENOTSUP) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "xattrop not supported by %s",
+                                        priv->children[child_index]->name);
+                                local->op_ret = -1;
+
+                        } else if (!child_went_down (op_ret, op_errno)) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "xattrop failed on child %s: %s",
+                                        priv->children[child_index]->name,
+                                        strerror (op_errno));
+                        }
+                        local->op_errno = op_errno;
+                        break;
+                }
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->child_up[child_index] = 0;
-			
-			if (op_errno == ENOTSUP) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"xattrop not supported by %s",
-					priv->children[child_index]->name);
-				local->op_ret = -1;
-			} else if (!child_went_down (op_ret, op_errno)) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"xattrop failed on child %s: %s",
-					priv->children[child_index]->name, 
-					strerror (op_errno));
-			}
-			local->op_errno = op_errno;
-		}
-
-		call_count = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (call_count == 0) {
-		if ((local->op_ret == -1) && 
-		    (local->op_errno == ENOTSUP)) {
-			local->transaction.resume (frame, this);
-		} else {
-			local->transaction.fop (frame, this);
-		}
-	}
+                call_count = --local->call_count;
+        }
+        UNLOCK (&frame->lock);
+
+        if (call_count == 0) {
+                if ((local->op_ret == -1) &&
+                    (local->op_errno == ENOTSUP)) {
+                        local->transaction.resume (frame, this);
+                } else {
+                        afr_transaction_perform_fop (frame, this);
+                }
+        }
 
-	return 0;	
+        return 0;
 }
 
-
-int 
+int
 afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv = this->private;
+        afr_private_t * priv = this->private;
+        int i = 0;
+        int ret = 0;
+        int call_count = 0;
+        dict_t **xattr = NULL;
+        afr_fd_ctx_t *fdctx = NULL;
+        afr_local_t *local = NULL;
+        int          piggyback = 0;
+        afr_internal_lock_t *int_lock = NULL;
+        unsigned char       *locked_nodes = NULL;
+
+        local = frame->local;
+        int_lock = &local->internal_lock;
+
+        xattr = alloca (priv->child_count * sizeof (*xattr));
+        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
+
+        for (i = 0; i < priv->child_count; i++) {
+                xattr[i] = dict_new ();
+        }
+
+        call_count = afr_changelog_pre_op_call_count (local->transaction.type,
+                                                      int_lock,
+                                                      priv->child_count);
+        if (call_count == 0) {
+                local->internal_lock.lock_cbk =
+                        local->transaction.done;
+                afr_unlock (frame, this);
+                goto out;
+        }
 
-	int i = 0;				
-	int ret = 0;
-	int call_count = 0;		     
-	dict_t *xattr = NULL;
+        local->call_count = call_count;
+
+        __mark_all_pending (local->pending, priv->child_count,
+                            local->transaction.type);
+
+        if (local->fd)
+                fdctx = afr_fd_ctx_get (local->fd, this);
+
+        locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
+        for (i = 0; i < priv->child_count; i++) {
+                if (!locked_nodes[i])
+                        continue;
+                ret = afr_set_pending_dict (priv, xattr[i], local->pending,
+                                            i, LOCAL_FIRST);
+
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_INFO,
+                                "failed to set pending entry");
+
+
+                switch (local->transaction.type) {
+                case AFR_DATA_TRANSACTION:
+                {
+                        if (!fdctx) {
+                                STACK_WIND_COOKIE (frame,
+                                                   afr_changelog_pre_op_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[i],
+                                                   priv->children[i]->fops->xattrop,
+                                                   &(local->loc),
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
+                                break;
+                        }
+
+                        LOCK (&local->fd->lock);
+                        {
+                                piggyback = 0;
+                                if (fdctx->pre_op_done[i]) {
+                                        fdctx->pre_op_piggyback[i]++;
+                                        piggyback = 1;
+                                        fdctx->hit++;
+                                } else {
+                                        fdctx->miss++;
+                                }
+                        }
+                        UNLOCK (&local->fd->lock);
+
+                        afr_set_delayed_post_op (frame, this);
+
+                        if (piggyback)
+                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
+                        else
+                                STACK_WIND_COOKIE (frame,
+                                                   afr_changelog_pre_op_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[i],
+                                                   priv->children[i]->fops->fxattrop,
+                                                   local->fd,
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
+                }
+                break;
+                case AFR_METADATA_TRANSACTION:
+                {
+                        if (local->optimistic_change_log) {
+                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
+                                break;
+                        }
+
+                        if (local->fd)
+                                STACK_WIND_COOKIE (frame,
+                                                   afr_changelog_pre_op_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[i],
+                                                   priv->children[i]->fops->fxattrop,
+                                                   local->fd,
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
+                        else
+                                STACK_WIND_COOKIE (frame,
+                                                   afr_changelog_pre_op_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[i],
+                                                   priv->children[i]->fops->xattrop,
+                                                   &(local->loc),
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
+                }
+                break;
+
+                case AFR_ENTRY_RENAME_TRANSACTION:
+                {
+                        if (local->optimistic_change_log) {
+                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
+                        } else {
+                                STACK_WIND_COOKIE (frame,
+                                                   afr_changelog_pre_op_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[i],
+                                                   priv->children[i]->fops->xattrop,
+                                                   &local->transaction.new_parent_loc,
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
+                        }
+
+                        call_count--;
+                }
 
-	afr_local_t *local = NULL;
 
-	local = frame->local;
-	xattr = get_new_dict ();
-	dict_ref (xattr);
+                /*
+                  set it again because previous stack_wind
+                  might have already returned (think of case
+                  where subvolume is posix) and would have
+                  used the dict as placeholder for return
+                  value
+                */
+
+                ret = afr_set_pending_dict (priv, xattr[i], local->pending,
+                                            i, LOCAL_FIRST);
+
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_INFO,
+                                "failed to set pending entry");
+
+                /* fall through */
+
+                case AFR_ENTRY_TRANSACTION:
+                {
+                        if (local->optimistic_change_log) {
+                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
+                                                          this, 1, 0, xattr[i],
+                                                          NULL);
+                                break;
+                        }
+
+                        if (local->fd)
+                                STACK_WIND_COOKIE (frame,
+                                                   afr_changelog_pre_op_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[i],
+                                                   priv->children[i]->fops->fxattrop,
+                                                   local->fd,
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
+                        else
+                                STACK_WIND_COOKIE (frame,
+                                                   afr_changelog_pre_op_cbk,
+                                                   (void *) (long) i,
+                                                   priv->children[i],
+                                                   priv->children[i]->fops->xattrop,
+                                                   &local->transaction.parent_loc,
+                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
+                                                   NULL);
+                }
+                break;
+                }
 
-	call_count = afr_up_children_count (priv->child_count, 
-					    local->child_up); 
+                if (!--call_count)
+                        break;
+        }
+out:
+        for (i = 0; i < priv->child_count; i++) {
+                dict_unref (xattr[i]);
+        }
 
-	if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-		call_count *= 2;
-	}
+        return 0;
+}
 
-	if (call_count == 0) {
-		/* no child is up */
-		dict_unref (xattr);
-		afr_unlock (frame, this);
-		return 0;
-	}
 
-	local->call_count = call_count;		
-
-	__mark_all_pending (local->pending_array, priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			ret = dict_set_static_bin (xattr, 
-						   local->transaction.pending, 
-						   local->pending_array, 
-						   (priv->child_count * 
-						    sizeof (int32_t)));
-			if (ret < 0)
-				gf_log (this->name, GF_LOG_ERROR, 
-					"failed to set pending entry");
-
-
-			switch (local->transaction.type) {
-			case AFR_DATA_TRANSACTION:
-			case AFR_METADATA_TRANSACTION:
-			case AFR_FLUSH_TRANSACTION:
-			{
-				if (local->fd)
-					STACK_WIND_COOKIE (frame, 
-							   afr_changelog_pre_op_cbk,
-							   (void *) (long) i,
-							   priv->children[i], 
-							   priv->children[i]->fops->fxattrop,
-							   local->fd,
-							   GF_XATTROP_ADD_ARRAY, xattr);
-				else
-					STACK_WIND_COOKIE (frame, 
-							   afr_changelog_pre_op_cbk,
-							   (void *) (long) i,
-							   priv->children[i], 
-							   priv->children[i]->fops->xattrop,
-							   &(local->loc), 
-							   GF_XATTROP_ADD_ARRAY, xattr);
-			}
-			break;
-				
-			case AFR_ENTRY_RENAME_TRANSACTION: 
-			{
-				STACK_WIND_COOKIE (frame, 
-						   afr_changelog_pre_op_cbk,
-						   (void *) (long) i,
-						   priv->children[i], 
-						   priv->children[i]->fops->xattrop,
-						   &local->transaction.new_parent_loc, 
-						   GF_XATTROP_ADD_ARRAY, xattr);
-
-				call_count--;
-			}
-
-
-			/* 
-			   set it again because previous stack_wind
-			   might have already returned (think of case
-			   where subvolume is posix) and would have
-			   used the dict as placeholder for return
-			   value
-			*/
-
-			ret = dict_set_static_bin (xattr, local->transaction.pending, 
-						   local->pending_array, 
-						   priv->child_count * sizeof (int32_t));
-			if (ret < 0)
-				gf_log (this->name, GF_LOG_ERROR, 
-					"failed to set pending entry");
-
-			/* fall through */
-				
-			case AFR_ENTRY_TRANSACTION:
-			{
-				if (local->fd)
-					STACK_WIND_COOKIE (frame, 
-							   afr_changelog_pre_op_cbk,
-							   (void *) (long) i,
-							   priv->children[i], 
-							   priv->children[i]->fops->fxattrop,
-							   local->fd, 
-							   GF_XATTROP_ADD_ARRAY, xattr);
-				else
-					STACK_WIND_COOKIE (frame, 
-							   afr_changelog_pre_op_cbk,
-							   (void *) (long) i,
-							   priv->children[i], 
-							   priv->children[i]->fops->xattrop,
-							   &local->transaction.parent_loc, 
-							   GF_XATTROP_ADD_ARRAY, xattr);
-			}
-
-			break;
-			}
-
-			if (!--call_count)
-				break;
-		}
-	}
+int
+afr_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
 
-	dict_unref (xattr);
-	return 0;
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Blocking inodelks failed.");
+                local->transaction.done (frame, this);
+        } else {
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Blocking inodelks done. Proceeding to FOP");
+                afr_internal_lock_finish (frame, this);
+        }
+
+        return 0;
 }
 
+
+int
+afr_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        /* Initiate blocking locks if non-blocking has failed */
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Non blocking inodelks failed. Proceeding to blocking");
+                int_lock->lock_cbk = afr_post_blocking_inodelk_cbk;
+                afr_blocking_lock (frame, this);
+        } else {
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Non blocking inodelks done. Proceeding to FOP");
+                afr_internal_lock_finish (frame, this);
+        }
+
+        return 0;
+}
+
+
+int
+afr_post_blocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Blocking entrylks failed.");
+                local->transaction.done (frame, this);
+        } else {
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Blocking entrylks done. Proceeding to FOP");
+                afr_internal_lock_finish (frame, this);
+        }
+
+        return 0;
+}
+
+
+int
+afr_post_nonblocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+
+        local = frame->local;
+        int_lock = &local->internal_lock;
+
+        /* Initiate blocking locks if non-blocking has failed */
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Non blocking entrylks failed. Proceeding to blocking");
+                int_lock->lock_cbk = afr_post_blocking_entrylk_cbk;
+                afr_blocking_lock (frame, this);
+        } else {
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Non blocking entrylks done. Proceeding to FOP");
+                afr_internal_lock_finish (frame, this);
+        }
+
+        return 0;
+}
+
+
+int
+afr_post_blocking_rename_cbk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        if (int_lock->lock_op_ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Blocking entrylks failed.");
+                local->transaction.done (frame, this);
+        } else {
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Blocking entrylks done. Proceeding to FOP");
+                afr_internal_lock_finish (frame, this);
+        }
+        return 0;
+}
+
+
+int
+afr_post_lower_unlock_cbk (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        GF_ASSERT (!int_lock->higher_locked);
+
+        int_lock->lock_cbk = afr_post_blocking_rename_cbk;
+        afr_blocking_lock (frame, this);
+
+        return 0;
+}
+
+
+int
+afr_set_transaction_flock (afr_local_t *local)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
+
+        int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        inodelk->flock.l_len   = local->transaction.len;
+        inodelk->flock.l_start = local->transaction.start;
+        inodelk->flock.l_type  = F_WRLCK;
+
+        return 0;
+}
+
+int
+afr_lock_rec (call_frame_t *frame, xlator_t *this)
+{
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+
+        int_lock->transaction_lk_type = AFR_TRANSACTION_LK;
+        int_lock->domain = this->name;
+
+        switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+                afr_set_transaction_flock (local);
+
+                int_lock->lock_cbk = afr_post_nonblocking_inodelk_cbk;
+
+                afr_nonblocking_inodelk (frame, this);
+                break;
+
+        case AFR_ENTRY_RENAME_TRANSACTION:
+
+                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
+                afr_nonblocking_entrylk (frame, this);
+                break;
+
+        case AFR_ENTRY_TRANSACTION:
+                int_lock->lk_basename = local->transaction.basename;
+                if (&local->transaction.parent_loc)
+                        int_lock->lk_loc = &local->transaction.parent_loc;
+                else
+                        GF_ASSERT (local->fd);
+
+                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
+                afr_nonblocking_entrylk (frame, this);
+                break;
+        }
+
+        return 0;
+}
+
+
+int
+afr_lock (call_frame_t *frame, xlator_t *this)
+{
+        afr_set_lock_number (frame, this);
+
+        return afr_lock_rec (frame, this);
+}
+
+
 /* }}} */
 
-/* {{{ lock */
+int
+afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
+{
+        if (__fop_changelog_needed (frame, this)) {
+                afr_changelog_pre_op (frame, this);
+        } else {
+                afr_transaction_perform_fop (frame, this);
+        }
+
+        return 0;
+}
 
-static
-int afr_lock_rec (call_frame_t *frame, xlator_t *this, int child_index);
 
-int32_t
-afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int32_t op_ret, int32_t op_errno)
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv = NULL;
-	int done = 0;
-	int child_index = (long) cookie;
+        afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
 
-	int call_count = 0;
+        /* call this function from any of the related optimizations
+           which benefit from delaying post op are enabled, namely:
 
-	local = frame->local;
-	priv  = this->private;
+           - changelog piggybacking
+           - eager locking
+        */
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-			/* wait for the other lock to return */
-			call_count = --local->call_count;
-		}
-
-		if (op_ret == -1) {
-			if (op_errno == ENOSYS) {
-				/* return ENOTSUP */
-				gf_log (this->name, GF_LOG_ERROR,
-					"subvolume does not support locking. "
-					"please load features/posix-locks xlator on server");
-				local->op_ret   = op_ret;
-				done = 1;
-			}
-
-			local->child_up[child_index] = 0;
-			local->op_errno = op_errno;
-		}
-	}
-	UNLOCK (&frame->lock);
-	
-	if (call_count == 0) {
-		if ((local->op_ret == -1) &&
-		    (local->op_errno == ENOSYS)) {
-			afr_unlock (frame, this);
-		} else {
-			local->transaction.locked_nodes[child_index] = 1;
-			local->transaction.lock_count++;
-			afr_lock_rec (frame, this, child_index + 1);
-		}
-	}
+        priv = this->private;
+        if (!priv)
+                return;
+
+        if (!priv->post_op_delay_secs)
+                return;
+
+        local = frame->local;
+        if (!local->transaction.eager_lock_on)
+                return;
+
+        if (!local)
+                return;
+
+        if (!local->fd)
+                return;
 
-	return 0;
+        if (local->op == GF_FOP_WRITE)
+                local->delayed_post_op = _gf_true;
 }
 
+gf_boolean_t
+afr_are_multiple_fds_opened (inode_t *inode, xlator_t *this)
+{
+        afr_inode_ctx_t *ictx = NULL;
+
+        if (!inode) {
+                /* If false is returned, it may keep on taking eager-lock
+                 * which may lead to starvation, so return true to avoid that.
+                 */
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid inode");
+                return _gf_true;
+        }
+        /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+         * is taken mount2 opened the same file, it won't be able to
+         * perform any data operations until mount1 releases eager-lock.
+         * To avoid such scenario do not enable eager-lock for this transaction
+         * if open-fd-count is > 1
+         */
+
+        ictx = afr_inode_ctx_get (inode, this);
+        if (!ictx)
+                return _gf_true;
+
+        if (ictx->open_fd_count > 1)
+                return _gf_true;
 
-static loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
+        return _gf_false;
+}
+
+gf_boolean_t
+afr_any_fops_failed (afr_local_t *local, afr_private_t *priv)
 {
-	int ret = 0;
+        if (local->success_count != priv->child_count)
+                return _gf_true;
+        return _gf_false;
+}
+
+gf_boolean_t
+is_afr_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t      *local = NULL;
+        gf_boolean_t      res = _gf_false;
+        afr_private_t    *priv  = NULL;
+
+        priv  = this->private;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        if (!local->delayed_post_op)
+                goto out;
+
+        //Mark pending changelog ASAP
+        if (afr_any_fops_failed (local, priv))
+                goto out;
 
-	ret = strcmp (l1->path, l2->path);
-	
-	if (ret == 0) 
-		ret = strcmp (b1, b2);
+        if (local->fd && afr_are_multiple_fds_opened (local->fd->inode, this))
+                goto out;
 
-	if (ret <= 0)
-		return l1;
-	else
-		return l2;
+        res = _gf_true;
+out:
+        return res;
 }
 
 
-static
-int afr_lock_rec (call_frame_t *frame, xlator_t *this, int child_index)
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub);
+
+void
+afr_delayed_changelog_wake_up_cbk (void *data)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        fd_t           *fd = NULL;
 
-	struct flock flock;
+        fd = data;
 
-	loc_t * lower  = NULL;
-	loc_t * higher = NULL;
+        afr_delayed_changelog_wake_up (THIS, fd);
+}
 
-	const char *lower_name  = NULL;
-	const char *higher_name = NULL;
 
-	local = frame->local;
-	priv  = this->private;
+/*
+  Check if the frame is destined to get optimized away
+  with changelog piggybacking
+*/
+static gf_boolean_t
+is_piggyback_post_op (call_frame_t *frame, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        afr_local_t *local = NULL;
+        gf_boolean_t piggyback = _gf_true;
+        afr_private_t *priv = NULL;
+        int i = 0;
+
+        priv = frame->this->private;
+        local = frame->local;
+        fdctx = afr_fd_ctx_get (fd, frame->this);
+
+        LOCK(&fd->lock);
+        {
+                piggyback = _gf_true;
+
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!local->transaction.pre_op[i])
+                                continue;
+                        if (fdctx->pre_op_piggyback[i]) {
+                                fdctx->pre_op_piggyback[i]--;
+                                local->transaction.postop_piggybacked[i] = 1;
+                        } else {
+                                /* For at least _one_ subvolume we cannot
+                                   piggyback on the changelog, and have to
+                                   perform a hard POST-OP and therefore fsync
+                                   if necesssary
+                                */
+                                piggyback = _gf_false;
+                                GF_ASSERT (fdctx->pre_op_done[i]);
+                                fdctx->pre_op_done[i]--;
+                        }
+                }
+        }
+        UNLOCK(&fd->lock);
 
-	flock.l_start = local->transaction.start;
-	flock.l_len   = local->transaction.len;
-	flock.l_type  = F_WRLCK;
+        if (!afr_txn_nothing_failed (frame, frame->this)) {
+                /* something failed in this transaction,
+                   we will be performing a hard post-op
+                */
+                return _gf_false;
+        }
 
-	/* skip over children that are down */
-	while ((child_index < priv->child_count)
-	       && !local->child_up[child_index])
-		child_index++;
+        return piggyback;
+}
 
-	if ((child_index == priv->child_count) &&
-	    local->transaction.lock_count == 0) {
 
-		gf_log (this->name, GF_LOG_DEBUG,
-			"unable to lock on even one child");
+/* SET operation */
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
 
-		local->op_ret   = -1;
-		local->op_errno = EAGAIN;
+        fdctx = afr_fd_ctx_get (fd, this);
 
-		local->transaction.done (frame, this);
-		
-		return 0;
+        LOCK(&fd->lock);
+        {
+                fdctx->witnessed_unstable_write = _gf_true;
+        }
+        UNLOCK(&fd->lock);
 
-	}
+        return 0;
+}
 
-	if ((child_index == priv->child_count) 
-	    || (local->transaction.lock_count == 
-		afr_lock_server_count (priv, local->transaction.type))) {
+/* TEST and CLEAR operation */
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        gf_boolean_t witness = _gf_false;
 
-		/* we're done locking */
+	fdctx = afr_fd_ctx_get (fd, this);
+        if (!fdctx)
+                return _gf_true;
 
-		if (__changelog_needed_pre_op (frame, this)) {
-			afr_changelog_pre_op (frame, this);
-		} else {
-			local->transaction.fop (frame, this);
-		}
+        LOCK(&fd->lock);
+        {
+                if (fdctx->witnessed_unstable_write) {
+                        witness = _gf_true;
+                        fdctx->witnessed_unstable_write = _gf_false;
+                }
+        }
+        UNLOCK (&fd->lock);
 
-		return 0;
-	}
+        return witness;
+}
 
-	switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:		
-	case AFR_METADATA_TRANSACTION:
-	case AFR_FLUSH_TRANSACTION:
-
-		if (local->fd) {
-			STACK_WIND_COOKIE (frame, afr_lock_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index], 
-					   priv->children[child_index]->fops->finodelk,
-					   local->fd, F_SETLKW, &flock);
-			
-		} else {
-			STACK_WIND_COOKIE (frame, afr_lock_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index], 
-					   priv->children[child_index]->fops->inodelk,
-					   &local->loc, F_SETLKW, &flock);
-		}
-		
-		break;
-		
-	case AFR_ENTRY_RENAME_TRANSACTION:
+
+int
+afr_changelog_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *pre,
+                         struct iatt *post, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
+        afr_local_t *local = NULL;
+
+        priv = this->private;
+        local = frame->local;
+
+        if (afr_fop_failed (op_ret, op_errno)) {
+                /* Failure of fsync() is as good as failure of previous
+                   write(). So treat it like one.
+                */
+                gf_log (this->name, GF_LOG_WARNING,
+                        "fsync(%s) failed on subvolume %s. Transaction was %s",
+                        uuid_utoa (local->fd->inode->gfid),
+                        priv->children[child_index]->name,
+                        gf_fop_list[local->op]);
+
+                afr_transaction_fop_failed (frame, this, child_index);
+        }
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0)
+                afr_changelog_post_op_now (frame, this);
+
+        return 0;
+}
+
+
+int
+afr_changelog_fsync (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        int i = 0;
+        int call_count = 0;
+        afr_private_t *priv = NULL;
+ 	dict_t *xdata = NULL;
+ 	GF_UNUSED int ret = -1;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
+                                                     priv->child_count);
+
+        if (!call_count) {
+                /* will go straight to unlock */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+	xdata = dict_new();
+	if (xdata)
+		ret = dict_set_int32 (xdata, "batch-fsync", 1);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_changelog_fsync_cbk,
+                                (void *) (long) i, priv->children[i],
+                                priv->children[i]->fops->fsync, local->fd,
+                                1, xdata);
+                if (!--call_count)
+                        break;
+        }
+
+	if (xdata)
+		dict_unref (xdata);
+
+        return 0;
+}
+
+
+        int
+afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+	local = frame->local;
+        priv = this->private;
+
+        if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        if (is_piggyback_post_op (frame, local->fd)) {
+                /* just detected that this post-op is about to
+                   be optimized away as a new write() has
+                   already piggybacked on this frame's changelog.
+                   */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Calling afr_changelog_post_op_now() now will result in
+           issuing ->[f]xattrop().
+
+           Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+           responsible operation that what it might appear on the surface.
+
+           The changelog of a file (in the xattr of the file on the server)
+           stores information (pending count) about the state of the file
+           on the OTHER server. This changelog is blindly trusted, and must
+           therefore be updated in such a way it remains trustworthy. This
+           implies that decrementing the pending count (essentially "clearing
+           the dirty flag") must be done STRICTLY after we are sure that the
+           operation on the other server has reached stable storage.
+
+           While the backend filesystem on that server will eventually flush
+           it to stable storage, we (being in userspace) have no mechanism
+           to get notified when the write became "stable".
+
+           This means we need take matter into our own hands and issue an
+           fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+           and get an acknowledgement for it. And we need to wait for the
+           fsync() acknowledgement before initiating the hard POST-OP.
+
+           However if the FD itself was opened in O_SYNC or O_DSYNC then
+           we are already guaranteed that the writes were made stable as
+           part of the FOP itself. The same holds true for NFS stable
+           writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+           flag set in the writev() @flags param. For all other write types,
+           mark a flag in the fdctx whenever an unstable write is witnessed.
+           */
+
+        if (!afr_fd_has_witnessed_unstable_write (this, local->fd)) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Check whether users want durability and perform fsync/post-op
+         * accordingly.
+         */
+        if (priv->ensure_durability) {
+                /* Time to fsync() */
+                afr_changelog_fsync (frame, this);
+        } else {
+                afr_changelog_post_op_now (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub)
+{
+	afr_fd_ctx_t      *fd_ctx = NULL;
+	call_frame_t      *prev_frame = NULL;
+	struct timespec    delta = {0, };
+	afr_private_t     *priv = NULL;
+	afr_local_t       *local = NULL;
+
+	priv = this->private;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+                goto out;
+
+	delta.tv_sec = priv->post_op_delay_secs;
+	delta.tv_nsec = 0;
+
+	pthread_mutex_lock (&fd_ctx->delay_lock);
 	{
-		local->call_count = 2;
-
-		lower = lower_path (&local->transaction.parent_loc, 
-				    local->transaction.basename,
-				    &local->transaction.new_parent_loc,
-				    local->transaction.new_basename);
-		
-		lower_name = (lower == &local->transaction.parent_loc ? 
-			      local->transaction.basename :
-			      local->transaction.new_basename);
-
-		higher = (lower == &local->transaction.parent_loc ? 
-			  &local->transaction.new_parent_loc :
-			  &local->transaction.parent_loc);
-
-		higher_name = (higher == &local->transaction.parent_loc ? 
-			       local->transaction.basename :
-			       local->transaction.new_basename);
-
-
-		/* TODO: these locks should be blocking */
-
-		STACK_WIND_COOKIE (frame, afr_lock_cbk,
-				   (void *) (long) child_index,
-				   priv->children[child_index], 
-				   priv->children[child_index]->fops->entrylk, 
-				   lower, lower_name,
-				   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-		STACK_WIND_COOKIE (frame, afr_lock_cbk,
-				   (void *) (long) child_index,
-				   priv->children[child_index], 
-				   priv->children[child_index]->fops->entrylk, 
-				   higher, higher_name,
-				   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-		break;
+		prev_frame = fd_ctx->delay_frame;
+		fd_ctx->delay_frame = NULL;
+		if (fd_ctx->delay_timer)
+			gf_timer_call_cancel (this->ctx, fd_ctx->delay_timer);
+		fd_ctx->delay_timer = NULL;
+		if (!frame)
+			goto unlock;
+		fd_ctx->delay_timer = gf_timer_call_after (this->ctx, delta,
+							   afr_delayed_changelog_wake_up_cbk,
+							   fd);
+		fd_ctx->delay_frame = frame;
 	}
-		
-	case AFR_ENTRY_TRANSACTION:
-		if (local->fd) {
-			STACK_WIND_COOKIE (frame, afr_lock_cbk,
-					   (void *) (long) child_index,	
-					   priv->children[child_index], 
-					   priv->children[child_index]->fops->fentrylk, 
-					   local->fd, 
-					   local->transaction.basename,
-					   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-		} else {
-			STACK_WIND_COOKIE (frame, afr_lock_cbk,
-					   (void *) (long) child_index,	
-					   priv->children[child_index], 
-					   priv->children[child_index]->fops->entrylk, 
-					   &local->transaction.parent_loc, 
-					   local->transaction.basename,
-					   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-		}
-
-		break;
+unlock:
+	pthread_mutex_unlock (&fd_ctx->delay_lock);
+
+out:
+	if (prev_frame) {
+		local = prev_frame->local;
+		local->transaction.resume_stub = stub;
+		afr_changelog_post_op_safe (prev_frame, this);
+	} else if (stub) {
+		call_resume (stub);
 	}
+}
+
 
-	return 0;
+void
+afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (is_afr_delayed_changelog_post_op_needed (frame, this))
+                afr_delayed_changelog_post_op (this, frame, local->fd, NULL);
+        else
+                afr_changelog_post_op_safe (frame, this);
 }
 
 
-int32_t afr_lock (call_frame_t *frame, xlator_t *this)
+
+/* Wake up the sleeping/delayed post-op, and also register
+   a stub to have it resumed after this transaction
+   completely finishes.
+
+   The @stub gets saved in @local and gets resumed in
+   afr_local_cleanup()
+   */
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
 {
-	return afr_lock_rec (frame, this, 0);
+        afr_delayed_changelog_post_op (this, NULL, fd, stub);
 }
 
 
-/* }}} */
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, NULL);
+}
 
-int32_t
+
+        int
 afr_transaction_resume (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
+
+        local    = frame->local;
+        int_lock = &local->internal_lock;
+        priv     = this->private;
+
+        if (local->transaction.eager_lock_on) {
+                /* We don't need to retain "local" in the
+                   fd list anymore, writes to all subvols
+                   are finished by now */
+                LOCK (&local->fd->lock);
+                {
+                        list_del_init (&local->transaction.eager_locked);
+                }
+                UNLOCK (&local->fd->lock);
+        }
 
-	local = frame->local;
-	priv  = this->private;
-
-	if (__changelog_needed_post_op (frame, this)) {
-		afr_changelog_post_op (frame, this);
-	} else {
-		if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-			local->transaction.done (frame, this);
-		} else {
-			afr_unlock (frame, this);
-		}
-	}
+        afr_restore_lk_owner (frame);
+
+        if (__fop_changelog_needed (frame, this)) {
+                afr_changelog_post_op (frame, this);
+        } else {
+                if (afr_lock_server_count (priv, local->transaction.type) == 0) {
+                        local->transaction.done (frame, this);
+                } else {
+                        int_lock->lock_cbk = local->transaction.done;
+                        afr_unlock (frame, this);
+                }
+        }
 
-	return 0;
+        return 0;
 }
 
 
 /**
- * afr_transaction_child_died - inform that a child died during an fop
+ * afr_transaction_fop_failed - inform that an fop failed
  */
 
 void
-afr_transaction_child_died (call_frame_t *frame, xlator_t *this, int child_index)
+afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
+                            int child_index)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
 
-	local = frame->local;
-	priv  = this->private;
+        local = frame->local;
+        priv  = this->private;
 
-	__mark_child_dead (local->pending_array, priv->child_count, child_index);
+        __mark_child_dead (local->pending, priv->child_count,
+                        child_index, local->transaction.type);
 }
 
 
-int32_t
+
+        static gf_boolean_t
+afr_locals_overlap (afr_local_t *local1, afr_local_t *local2)
+{
+        uint64_t start1 = local1->transaction.start;
+        uint64_t start2 = local2->transaction.start;
+        uint64_t end1 = 0;
+        uint64_t end2 = 0;
+
+        if (local1->transaction.len)
+                end1 = start1 + local1->transaction.len - 1;
+        else
+                end1 = ULLONG_MAX;
+
+        if (local2->transaction.len)
+                end2 = start2 + local2->transaction.len - 1;
+        else
+                end2 = ULLONG_MAX;
+
+        return ((end1 >= start2) && (end2 >= start1));
+}
+
+void
+afr_transaction_eager_lock_init (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_fd_ctx_t  *fdctx = NULL;
+        afr_local_t   *each = NULL;
+
+        priv = this->private;
+
+        if (!local->fd)
+                return;
+
+        if (local->transaction.type != AFR_DATA_TRANSACTION)
+                return;
+
+        if (!priv->eager_lock)
+                return;
+
+        fdctx = afr_fd_ctx_get (local->fd, this);
+        if (!fdctx)
+                return;
+
+        if (afr_are_multiple_fds_opened (local->fd->inode, this))
+                return;
+        /*
+         * Once full file lock is acquired in eager-lock phase, overlapping
+         * writes do not compete for inode-locks, instead are transferred to the
+         * next writes. Because of this overlapping writes are not ordered.
+         * This can cause inconsistencies in replication.
+         * Example:
+         * Two overlapping writes w1, w2 are sent in parallel on same fd
+         * in two threads t1, t2.
+         * Both threads can execute afr_writev_wind in the following manner.
+         * t1 winds w1 on brick-0
+         * t2 winds w2 on brick-0
+         * t2 winds w2 on brick-1
+         * t1 winds w1 on brick-1
+         *
+         * This check makes sure the locks are not transferred for
+         * overlapping writes.
+         */
+        LOCK (&local->fd->lock);
+        {
+                list_for_each_entry (each, &fdctx->eager_locked,
+                                     transaction.eager_locked) {
+                        if (afr_locals_overlap (each, local)) {
+                                local->transaction.eager_lock_on = _gf_false;
+                                goto unlock;
+                        }
+                }
+
+                local->transaction.eager_lock_on = _gf_true;
+                list_add_tail (&local->transaction.eager_locked,
+                               &fdctx->eager_locked);
+        }
+unlock:
+        UNLOCK (&local->fd->lock);
+}
+
+
+int
 afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        fd_t            *fd   = NULL;
+        int             ret   = -1;
 
-	local = frame->local;
-	priv  = this->private;
+        local = frame->local;
+        priv  = this->private;
 
-	afr_transaction_local_init (local, priv);
+        local->transaction.resume = afr_transaction_resume;
+        local->transaction.type   = type;
 
-	local->transaction.resume = afr_transaction_resume;
-	local->transaction.type   = type;
+        ret = afr_transaction_local_init (local, this);
+        if (ret < 0)
+            goto out;
 
-	if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-		if (__changelog_needed_pre_op (frame, this)) {
-			afr_changelog_pre_op (frame, this);
-		} else {
-			local->transaction.fop (frame, this);
-		}
-	} else {
-		afr_lock (frame, this);
-	}
+        afr_transaction_eager_lock_init (local, this);
 
-	return 0;
+        if (local->fd && local->transaction.eager_lock_on)
+                afr_set_lk_owner (frame, this, local->fd);
+        else
+                afr_set_lk_owner (frame, this, frame->root);
+
+        if (!local->transaction.eager_lock_on && local->loc.inode) {
+                fd = fd_lookup (local->loc.inode, frame->root->pid);
+                if (fd == NULL)
+                        fd = fd_lookup_anonymous (local->loc.inode);
+
+                if (fd) {
+                        afr_delayed_changelog_wake_up (this, fd);
+                        fd_unref (fd);
+                }
+        }
+
+        if (afr_lock_server_count (priv, local->transaction.type) == 0) {
+                afr_internal_lock_finish (frame, this);
+        } else {
+                afr_lock (frame, this);
+        }
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index 49cdd219f..fa626fd0d 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -1,36 +1,51 @@
 /*
-   Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __TRANSACTION_H__
 #define __TRANSACTION_H__
 
-#define AFR_METADATA_PENDING "trusted.glusterfs.afr.metadata-pending"
-
-#define AFR_DATA_PENDING "trusted.glusterfs.afr.data-pending"
-
-#define AFR_ENTRY_PENDING "trusted.glusterfs.afr.entry-pending"
+typedef enum {
+        LOCAL_FIRST = 1,
+        LOCAL_LAST = 2
+} afr_xattrop_type_t;
 
 void
-afr_transaction_child_died (call_frame_t *frame, xlator_t *this,
+afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 			    int child_index);
 
+int
+afr_lock_server_count (afr_private_t *priv, afr_transaction_type type);
+
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom);
+
 int32_t
 afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+int
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
+                      int child, afr_xattrop_type_t op);
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this);
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+
+void
+__mark_all_success (int32_t *pending[], int child_count,
+                    afr_transaction_type type);
+gf_boolean_t
+afr_any_fops_failed (afr_local_t *local, afr_private_t *priv);
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this);
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index e627da1fb..c724eb2ae 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007, 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -28,2324 +19,775 @@
 #define _CONFIG_H
 #include "config.h"
 #endif
+#include "afr-common.c"
 
-#include "glusterfs.h"
-#include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-inode-read.h"
-#include "afr-inode-write.h"
-#include "afr-dir-read.h"
-#include "afr-dir-write.h"
-#include "afr-transaction.h"
-
-#include "afr-self-heal.h"
-
-
-/**
- * afr_local_cleanup - cleanup everything in frame->local
- */
-
-void
-afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
-{
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-
-
-	sh = &local->self_heal;
-	priv = this->private;
-
-	if (sh->buf)
-		FREE (sh->buf);
-
-	if (sh->xattr) {
-		for (i = 0; i < priv->child_count; i++) {
-			if (sh->xattr[i]) {
-				dict_unref (sh->xattr[i]);
-				sh->xattr[i] = NULL;
-			}
-		}
-		FREE (sh->xattr);
-	}
-
-	if (sh->child_errno)
-		FREE (sh->child_errno);
-
-	if (sh->pending_matrix) {
-		for (i = 0; i < priv->child_count; i++) {
-			FREE (sh->pending_matrix[i]);
-		}
-		FREE (sh->pending_matrix);
-	}
-
-	if (sh->delta_matrix) {
-		for (i = 0; i < priv->child_count; i++) {
-			FREE (sh->delta_matrix[i]);
-		}
-		FREE (sh->delta_matrix);
-	}
-
-	if (sh->sources)
-		FREE (sh->sources);
-
-	if (sh->success)
-		FREE (sh->success);
-
-	if (sh->healing_fd) {
-		fd_unref (sh->healing_fd);
-		sh->healing_fd = NULL;
-	}
-
-	loc_wipe (&sh->parent_loc);
-}
-
-
-void 
-afr_local_cleanup (afr_local_t *local, xlator_t *this)
-{
-	if (!local)
-		return;
-
-	afr_local_sh_cleanup (local, this);
-
-	FREE (local->child_errno);
-	FREE (local->pending_array);
-
-	loc_wipe (&local->loc);
-	loc_wipe (&local->newloc);
-
-	FREE (local->transaction.locked_nodes);
-	FREE (local->transaction.child_errno);
-
-	FREE (local->transaction.basename);
-	FREE (local->transaction.new_basename);
-
-	loc_wipe (&local->transaction.parent_loc);	
-	loc_wipe (&local->transaction.new_parent_loc);
-
-	if (local->fd)
-		fd_unref (local->fd);
-	
-	if (local->xattr_req)
-		dict_unref (local->xattr_req);
-
-	FREE (local->child_up);
-
-	{ /* lookup */
-		if (local->cont.lookup.xattr)
-			dict_unref (local->cont.lookup.xattr);
-	}
-
-	{ /* getxattr */
-		if (local->cont.getxattr.name)
-			FREE (local->cont.getxattr.name);
-	}
-
-	{ /* lk */
-		if (local->cont.lk.locked_nodes)
-			FREE (local->cont.lk.locked_nodes);
-	}
-
-	{ /* checksum */
-		if (local->cont.checksum.file_checksum)
-			FREE (local->cont.checksum.file_checksum);
-		if (local->cont.checksum.dir_checksum)
-			FREE (local->cont.checksum.dir_checksum);
-	}
-
-	{ /* create */
-		if (local->cont.create.fd)
-			fd_unref (local->cont.create.fd);
-	}
-
-	{ /* writev */
-		FREE (local->cont.writev.vector);
-	}
-
-	{ /* setxattr */
-		if (local->cont.setxattr.dict)
-			dict_unref (local->cont.setxattr.dict);
-	}
-
-	{ /* removexattr */
-		FREE (local->cont.removexattr.name);
-	}
-
-	{ /* symlink */
-		FREE (local->cont.symlink.linkpath);
-	}
-}
-
-
-int
-afr_frame_return (call_frame_t *frame)
-{
-	afr_local_t *local = NULL;
-	int          call_count = 0;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		call_count = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	return call_count;
-}
-
-/**
- * first_up_child - return the index of the first child that is up
- */
-
-int
-afr_first_up_child (afr_private_t *priv)
-{
-	xlator_t ** children = NULL;
-	int         ret      = -1;
-	int         i        = 0;
-
-	LOCK (&priv->lock);
-	{
-		children = priv->children;
-		for (i = 0; i < priv->child_count; i++) {
-			if (priv->child_up[i]) {
-				ret = i;
-				break;
-			}
-		}
-	}
-	UNLOCK (&priv->lock);
-
-	return ret;
-}
-
-
-/**
- * up_children_count - return the number of children that are up
- */
-
-int
-afr_up_children_count (int child_count, unsigned char *child_up)
-{
-	int i   = 0;
-	int ret = 0;
-
-	for (i = 0; i < child_count; i++)
-		if (child_up[i])
-			ret++;
-	return ret;
-}
-
-
-int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
-{
-	int ret = 0;
-	int i;
-
-	for (i = 0; i < child_count; i++)
-		if (locked_nodes[i])
-			ret++;
-
-	return ret;
-}
-
-
-ino64_t
-afr_itransform (ino64_t ino, int child_count, int child_index)
-{
-	ino64_t scaled_ino = -1;
-
-	if (ino == ((uint64_t) -1)) {
-		scaled_ino = ((uint64_t) -1);
-		goto out;
-	}
-
-	scaled_ino = (ino * child_count) + child_index;
-
-out:
-	return scaled_ino;
-}
-
-
-int
-afr_deitransform_orig (ino64_t ino, int child_count)
-{
-	int index = -1;
-
-	index = ino % child_count;
-
-	return index;
-}
-
-
-int
-afr_deitransform (ino64_t ino, int child_count)
-{
-	return 0;
-}
-
-
-int
-afr_self_heal_cbk (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	int ret = -1;
-
-	local = frame->local;
-
-	if (local->govinda_gOvinda) {
-		ret = inode_ctx_put (local->cont.lookup.inode, this, 1);
-
-		if (ret < 0) {
-			local->op_ret   = -1;
-			local->op_errno = -ret;
-		}
-	} else {
-		inode_ctx_del (local->cont.lookup.inode, this, NULL);
-	}
-
-	AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			  local->cont.lookup.inode,
-			  &local->cont.lookup.buf,
-			  local->cont.lookup.xattr);
-
-	return 0;
-}
-
-
-int
-afr_lookup_cbk (call_frame_t *frame, void *cookie,
-		xlator_t *this,	int32_t op_ret,	int32_t op_errno,
-		inode_t *inode,	struct stat *buf, dict_t *xattr)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	struct stat *   lookup_buf = NULL;
-	int             call_count = -1;
-	int             child_index = -1;
-	int             prev_child_index = -1;
-	uint32_t        open_fd_count = 0;
-	int             ret = 0;
-
-	child_index = (long) cookie;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-		local = frame->local;
-
-		lookup_buf = &local->cont.lookup.buf;
-
-		if (op_ret == -1) {
-			if (op_errno == ENOENT)
-				local->enoent_count++;
-			
-			if (op_errno != ENOTCONN)
-				local->op_errno = op_errno;
-
-			goto unlock;
-		}
-
-		if (afr_sh_has_metadata_pending (xattr, child_index, this))
-			local->need_metadata_self_heal = 1;
-
-		if (afr_sh_has_entry_pending (xattr, child_index, this))
-			local->need_entry_self_heal = 1;
-
-		if (afr_sh_has_data_pending (xattr, child_index, this))
-			local->need_data_self_heal = 1;
-
-		ret = dict_get_uint32 (xattr, GLUSTERFS_OPEN_FD_COUNT,
-				       &open_fd_count);
-		local->open_fd_count += open_fd_count;
-
-		/* in case of revalidate, we need to send stat of the
-		 * child whose stat was sent during the first lookup.
-		 * (so that time stamp does not vary with revalidate.
-		 * in case it is down, stat of the fist success will
-		 * be replied */
-
-		/* inode number should be preserved across revalidates */
-
-		if (local->success_count == 0) {
-			local->op_ret   = op_ret;
-				
-			local->cont.lookup.inode = inode;
-			local->cont.lookup.xattr = dict_ref (xattr);
-
-			*lookup_buf = *buf;
-			lookup_buf->st_ino = afr_itransform (buf->st_ino,
-							     priv->child_count,
-							     child_index);
-		} else {
-			if (FILETYPE_DIFFERS (buf, lookup_buf)) {
-				/* mismatching filetypes with same name
-				   -- Govinda !! GOvinda !!!
-				*/
-				local->govinda_gOvinda = 1;
-			}
-
-			if (PERMISSION_DIFFERS (buf, lookup_buf)) {
-				/* mismatching permissions */
-				local->need_metadata_self_heal = 1;
-			}
-
-			if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
-				/* mismatching permissions */
-				local->need_metadata_self_heal = 1;
-			}
-
-			if (SIZE_DIFFERS (buf, lookup_buf)
-			    && S_ISREG (buf->st_mode)) {
-				local->need_data_self_heal = 1;
-			}
-
-			prev_child_index = afr_deitransform_orig (lookup_buf->st_ino, 
-								  priv->child_count);
-			if (child_index < prev_child_index) {
-				*lookup_buf = *buf;
-				lookup_buf->st_ino = afr_itransform (buf->st_ino,
-								     priv->child_count,
-								     child_index);
-			}
-		}
-
-		local->success_count++;
-	}
-unlock:
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		if (local->op_ret == 0) {
-			/* KLUDGE: assuming DHT will not itransform in 
-			   revalidate */
-			if (local->cont.lookup.inode->ino)
-				lookup_buf->st_ino = 
-					local->cont.lookup.inode->ino;
-		}
-
-		if (local->success_count && local->enoent_count) {
-			local->need_metadata_self_heal = 1;
-			local->need_data_self_heal = 1;
-			local->need_entry_self_heal = 1;
-		}
-
-		if (local->success_count) {
-			/* check for govinda_gOvinda case in previous lookup */
-			if (!inode_ctx_get (local->cont.lookup.inode, 
-					   this, NULL))
-				local->need_data_self_heal = 1;
-		}
-
-		if ((local->need_metadata_self_heal
-		     || local->need_data_self_heal
-		     || local->need_entry_self_heal)
-		    && (!local->open_fd_count)) {
-
-			if (!local->cont.lookup.inode->st_mode) {
-				/* fix for RT #602 */
-				local->cont.lookup.inode->st_mode =
-					lookup_buf->st_mode;
-			}
-
-			afr_self_heal (frame, this, afr_self_heal_cbk);
-		} else {
-			AFR_STACK_UNWIND (frame, local->op_ret,
-					  local->op_errno,
-					  local->cont.lookup.inode, 
-					  &local->cont.lookup.buf,
-					  local->cont.lookup.xattr);
-		}
-	}
-
-	return 0;
-}
-
-
-int
-afr_lookup (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, dict_t *xattr_req)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t   *local = NULL;
-	int            ret = -1;
-	int            i = 0;
-	int32_t        op_errno = 0;
-
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	local->op_ret = -1;
-
-	frame->local = local;
-
-	loc_copy (&local->loc, loc);
-
-	local->reval_child_index = 0;
-
-	local->call_count = priv->child_count;
-
-	local->child_up = memdup (priv->child_up, priv->child_count);
-	local->child_count = afr_up_children_count (priv->child_count,
-						    local->child_up);
-
-	/* By default assume ENOTCONN. On success it will be set to 0. */
-	local->op_errno = ENOTCONN;
-	
-	if ((xattr_req == NULL)
-	    && (priv->metadata_self_heal
-		|| priv->data_self_heal
-		|| priv->entry_self_heal))
-		local->xattr_req = dict_new ();
-	else
-		local->xattr_req = dict_ref (xattr_req);
-
-	if (priv->metadata_self_heal) {
-		ret = dict_set_uint64 (local->xattr_req, AFR_METADATA_PENDING,
-				       priv->child_count * sizeof(int32_t));
-	}
-	
-	if (priv->data_self_heal) {
-		ret = dict_set_uint64 (local->xattr_req, AFR_DATA_PENDING,
-				       priv->child_count * sizeof(int32_t));
-	}
-	
-	if (priv->entry_self_heal) {
-		ret = dict_set_uint64 (local->xattr_req, AFR_ENTRY_PENDING,
-				       priv->child_count * sizeof(int32_t));
-	}
-
-	ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_OPEN_FD_COUNT, 0);
-
-	for (i = 0; i < priv->child_count; i++) {
-		STACK_WIND_COOKIE (frame, afr_lookup_cbk, (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->lookup,
-				   loc, local->xattr_req);
-	}
-
-	ret = 0;
-out:
-	if (ret == -1)
-		AFR_STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL, NULL);
-
-	return 0;
-}
-
-
-/* {{{ open */
-
-int
-afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-			int32_t op_ret, int32_t op_errno, struct stat *buf)
-{
-	afr_local_t * local = frame->local;
-
-	AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			  local->fd);
-	return 0;
-}
-
-
-int
-afr_open_cbk (call_frame_t *frame, void *cookie,
-	      xlator_t *this, int32_t op_ret, int32_t op_errno,
-	      fd_t *fd)
-{
-	afr_local_t *  local = NULL;
-	afr_private_t * priv = NULL;
-
-	int call_count = -1;
-	
-	priv  = this->private;
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		if ((local->cont.open.flags & O_TRUNC)
-		    && (local->op_ret >= 0)) {
-			STACK_WIND (frame, afr_open_ftruncate_cbk,
-				    this, this->fops->ftruncate,
-				    fd, 0);
-		} else {
-			AFR_STACK_UNWIND (frame, local->op_ret,
-					  local->op_errno, local->fd);
-		}
-	}
-
-	return 0;
-}
-
-
-int
-afr_open (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc, int32_t flags, fd_t *fd)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t *   local = NULL;
-	
-	int     i = 0;
-	int   ret = -1;
-
-	int32_t call_count = 0;	
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-	int32_t wind_flags = flags & (~O_TRUNC);
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-	VALIDATE_OR_GOTO (loc, out);
-	
-	priv = this->private;
-
-	ret = inode_ctx_get (loc->inode, this, NULL);
-	if (ret == 0) {
-		/* if ctx is set it means self-heal failed */
-
-		gf_log (this->name, GF_LOG_WARNING, 
-			"returning EIO, file has to be manually corrected "
-			"in backend");
-		op_errno = EIO;
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	frame->local = local;
-	call_count   = local->call_count;
-
-	local->cont.open.flags = flags;
-	local->fd = fd_ref (fd);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->open,
-					   loc, wind_flags, fd);
-			
-			if (!--call_count)
-				break;
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, fd);
-	}
-
-	return 0;
-}
-
-/* }}} */
+#define SHD_INODE_LRU_LIMIT          2048
+#define AFR_EH_HEALED_LIMIT          1024
+#define AFR_EH_HEAL_FAIL_LIMIT       1024
+#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
 
-/* {{{ flush */
+struct volume_options options[];
 
-int
-afr_flush_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		      int32_t op_ret, int32_t op_errno)
+int32_t
+notify (xlator_t *this, int32_t event,
+        void *data, ...)
 {
-	afr_local_t *   local = NULL;
-
-	int call_count  = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0)
-			local->op_ret = 0;
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        int ret = -1;
+        va_list         ap;
+        void *data2 = NULL;
 
-	call_count = afr_frame_return (frame);
+        va_start (ap, data);
+        data2 = va_arg (ap, dict_t*);
+        va_end (ap);
+        ret = afr_notify (this, event, data, data2);
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return ret;
 }
 
-
-int
-afr_flush_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	
-	int i = 0;
-	int call_count = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_flush_wind_cbk, 
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->flush,
-					   local->fd);
-		
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
-}
-
-
-int
-afr_flush_done (call_frame_t *frame, xlator_t *this)
+int32_t
+mem_acct_init (xlator_t *this)
 {
-	afr_local_t *local = NULL;
-
-	local = frame->local;
+        int     ret = -1;
 
-	AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-
-	return 0;
-}
+        if (!this)
+                return ret;
 
+        ret = xlator_mem_acct_init (this, gf_afr_mt_end + 1);
 
-int
-afr_simple_flush_cbk (call_frame_t *frame, void *cookie,
-		      xlator_t *this, int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t *local = NULL;
-        
-        int call_count = -1;
-	
-        local = frame->local;
-	
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
-
-                local->op_errno = op_errno;
+        if (ret != 0) {
+                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+                       "failed");
+                return ret;
         }
-        UNLOCK (&frame->lock);
-	
-        call_count = afr_frame_return (frame);
-	
-        if (call_count == 0)
-                AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	
-        return 0;
-}
-
-
-static int
-__is_fd_ctx_set (xlator_t *this, fd_t *fd)
-{
-	int _ret   = 0;
-	int op_ret = 0;
 
-	_ret = fd_ctx_get (fd, this, NULL);
-	if (_ret == 0)
-		op_ret = 1;
-
-	return op_ret;
+        return ret;
 }
 
 
 int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+xlator_subvolume_index (xlator_t *this, xlator_t *subvol)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-
-	int ret        = -1;
-	int i          = 0;
-	int call_count = 0;
-
-	int op_ret   = -1;
-	int op_errno = 0;
+        int index = -1;
+        int i = 0;
+        xlator_list_t *list = NULL;
 
-        int transaction_needed = 0;
+        list = this->children;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	frame->local = local;
-
-        LOCK (&fd->inode->lock);
-        {
-                if (__is_fd_ctx_set (this, fd)) {
-                        transaction_needed = 1;
-                        fd_ctx_del (fd, this, NULL);
-                } else {
-                        transaction_needed = 0;
+        while (list) {
+                if (subvol == list->xlator ||
+                    strcmp (subvol->name, list->xlator->name) == 0) {
+                        index = i;
+                        break;
                 }
+                list = list->next;
+                i++;
         }
-        UNLOCK (&fd->inode->lock);
-
-        if (transaction_needed) {
-		local->op = GF_FOP_FLUSH;
-		local->transaction.fop    = afr_flush_wind;
-		local->transaction.done   = afr_flush_done;
-		
-		local->fd                 = fd_ref (fd);
-		
-		local->transaction.start  = 0;
-		local->transaction.len    = 0;
-		
-		local->transaction.pending = AFR_DATA_PENDING;
-		
-		afr_transaction (frame, this, AFR_FLUSH_TRANSACTION);
-	} else {
-		/*
-		 * if fd's ctx is not set, then there is no need
-		 * to erase changelog. So just send the flush
-		 */
-
-		call_count = local->call_count;
-
-		for (i = 0; i < priv->child_count; i++) {
-			if (local->child_up[i]) {
-				STACK_WIND (frame, afr_simple_flush_cbk,
-					    priv->children[i],
-					    priv->children[i]->fops->flush,
-					    fd);
-
-				if (!--call_count)
-					break;
-			}
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
 
-	return 0;
+        return index;
 }
 
-/* }}} */
-
-/* {{{ fsync */
-
-int
-afr_fsync_cbk (call_frame_t *frame, void *cookie,
-	       xlator_t *this, int32_t op_ret, int32_t op_errno)
+void
+fix_quorum_options (xlator_t *this, afr_private_t *priv, char *qtype)
 {
-	afr_local_t *local = NULL;
-	
-	int call_count = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0)
-			local->op_ret = 0;
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-
-	return 0;
+        if (priv->quorum_count && strcmp(qtype,"fixed")) {
+                gf_log(this->name,GF_LOG_WARNING,
+                       "quorum-type %s overriding quorum-count %u",
+                       qtype, priv->quorum_count);
+        }
+        if (!strcmp(qtype,"none")) {
+                priv->quorum_count = 0;
+        }
+        else if (!strcmp(qtype,"auto")) {
+                priv->quorum_count = AFR_QUORUM_AUTO;
+        }
 }
 
-
 int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	   int32_t datasync)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	int ret = -1;
-
-	int i = 0;
-	int32_t call_count = 0;
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	call_count = local->call_count;
-	frame->local = local;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_fsync_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->fsync,
-				    fd, datasync);
-			if (!--call_count)
-				break;
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
-}
-
-/* }}} */
-
-/* {{{ fsync */
-
-int32_t
-afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno)
+reconfigure (xlator_t *this, dict_t *options)
 {
-	afr_local_t *local = NULL;
-	
-	int call_count = -1;
+        afr_private_t *priv        = NULL;
+        xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
+        int            ret         = -1;
+        int            index       = -1;
+        char          *qtype       = NULL;
 
-	local = frame->local;
+        priv = this->private;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0)
-			local->op_ret = 0;
+        GF_OPTION_RECONF ("background-self-heal-count",
+                          priv->background_self_heal_count, options, uint32,
+                          out);
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        GF_OPTION_RECONF ("metadata-self-heal",
+                          priv->metadata_self_heal, options, bool, out);
 
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-
-	return 0;
-}
-
-
-int32_t
-afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	      int32_t datasync)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
+        GF_OPTION_RECONF ("data-self-heal", priv->data_self_heal, options, str,
+                          out);
 
-	int ret = -1;
+        GF_OPTION_RECONF ("entry-self-heal", priv->entry_self_heal, options,
+                          bool, out);
 
-	int i = 0;
-	int32_t call_count = 0;
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
+        GF_OPTION_RECONF ("strict-readdir", priv->strict_readdir, options, bool,
+                          out);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        GF_OPTION_RECONF ("data-self-heal-window-size",
+                          priv->data_self_heal_window_size, options,
+                          uint32, out);
 
-	priv = this->private;
+        GF_OPTION_RECONF ("data-change-log", priv->data_change_log, options,
+                          bool, out);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        GF_OPTION_RECONF ("metadata-change-log",
+                          priv->metadata_change_log, options, bool, out);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        GF_OPTION_RECONF ("entry-change-log", priv->entry_change_log, options,
+                          bool, out);
 
-	call_count = local->call_count;
-	frame->local = local;
+        GF_OPTION_RECONF ("data-self-heal-algorithm",
+                          priv->data_self_heal_algorithm, options, str, out);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_fsync_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->fsyncdir,
-				    fd, datasync);
-			if (!--call_count)
-				break;
-		}
-	}
+        GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options, bool, out);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
-}
-
-/* }}} */
-
-/* {{{ xattrop */
-
-int32_t
-afr_xattrop_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno,
-		 dict_t *xattr)
-{
-	afr_local_t *local = NULL;
-	
-	int call_count = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0)
-			local->op_ret = 0;
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno, xattr);
-
-	return 0;
-}
+        GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
 
+        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
+                          options, uint32, out);
 
-int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	     gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local  = NULL;
-
-	int ret = -1;
-
-	int i = 0;
-	int32_t call_count = 0;
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	call_count = local->call_count;
-	frame->local = local;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_xattrop_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->xattrop,
-				    loc, optype, xattr);
-			if (!--call_count)
-				break;
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
-}
-
-/* }}} */
-
-/* {{{ fxattrop */
-
-int32_t
-afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *xattr)
-{
-	afr_local_t *local = NULL;
-	
-	int call_count = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0)
-			local->op_ret = 0;
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno, xattr);
-
-	return 0;
-}
-
-
-int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	      gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local  = NULL;
-
-	int ret = -1;
-
-	int i = 0;
-	int32_t call_count = 0;
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	call_count = local->call_count;
-	frame->local = local;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_fxattrop_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->fxattrop,
-				    fd, optype, xattr);
-			if (!--call_count)
-				break;
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
-}
-
-/* }}} */
-
-
-int32_t
-afr_inodelk_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno)
-		
-{
-	afr_local_t *local = NULL;
-	
-	int call_count = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0)
-			local->op_ret = 0;
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-
-	return 0;
-}
-
-
-int32_t
-afr_inodelk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	     int32_t cmd, struct flock *flock)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local  = NULL;
-
-	int ret = -1;
-
-	int i = 0;
-	int32_t call_count = 0;
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	call_count = local->call_count;
-	frame->local = local;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_inodelk_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->inodelk,
-				    loc, cmd, flock);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
-}
-
-
-int32_t
-afr_finodelk_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno)
-		
-{
-	afr_local_t *local = NULL;
-	
-	int call_count = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0)
-			local->op_ret = 0;
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-
-	return 0;
-}
-
-
-int32_t
-afr_finodelk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	      int32_t cmd, struct flock *flock)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local  = NULL;
-
-	int ret = -1;
-
-	int i = 0;
-	int32_t call_count = 0;
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	call_count = local->call_count;
-	frame->local = local;
+        if (read_subvol) {
+                index = xlator_subvolume_index (this, read_subvol);
+                if (index == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
+                                read_subvol->name);
+                        goto out;
+                }
+                priv->read_child = index;
+        }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_finodelk_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->finodelk,
-				    fd, cmd, flock);
+        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
 
-			if (!--call_count)
-				break;
-		}
-	}
+        if (read_subvol_index >-1) {
+                index=read_subvol_index;
+                if (index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                index);
+                        goto out;
+                }
+                priv->read_child = index;
+        }
 
-	op_ret = 0;
+        GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
+        GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
+        GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
+                          uint32, out);
+        fix_quorum_options(this,priv,qtype);
+        GF_OPTION_RECONF ("heal-timeout", priv->shd.timeout, options,
+                          int32, out);
+
+	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
+			  uint32, out);
+
+        GF_OPTION_RECONF (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size,
+                          options, size, out);
+        /* Reset this so we re-discover in case the topology changed.  */
+        GF_OPTION_RECONF ("readdir-failover", priv->readdir_failover, options,
+                          bool, out);
+        GF_OPTION_RECONF ("ensure-durability", priv->ensure_durability, options,
+                          bool, out);
+        priv->did_discovery = _gf_false;
+
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
-}
-
-
-int32_t
-afr_entrylk_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno)
-		
-{
-	afr_local_t *local = NULL;
-	
-	int call_count = -1;
-
-	local = frame->local;
+        return ret;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0)
-			local->op_ret = 0;
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-
-	return 0;
 }
 
 
-int32_t
-afr_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	     const char *basename, entrylk_cmd cmd, entrylk_type type)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local  = NULL;
-
-	int ret = -1;
-
-	int i = 0;
-	int32_t call_count = 0;
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	call_count = local->call_count;
-	frame->local = local;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_entrylk_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->entrylk,
-				    loc, basename, cmd, type);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
-}
-
-
-
-int32_t
-afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno)
-		
-{
-	afr_local_t *local = NULL;
-	
-	int call_count = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0)
-			local->op_ret = 0;
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-
-	return 0;
-}
+static const char *favorite_child_warning_str = "You have specified subvolume '%s' "
+        "as the 'favorite child'. This means that if a discrepancy in the content "
+        "or attributes (ownership, permission, etc.) of a file is detected among "
+        "the subvolumes, the file on '%s' will be considered the definitive "
+        "version and its contents will OVERWRITE the contents of the file on other "
+        "subvolumes. All versions of the file except that on '%s' "
+        "WILL BE LOST.";
 
 
 int32_t
-afr_fentrylk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	      const char *basename, entrylk_cmd cmd, entrylk_type type)
+init (xlator_t *this)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local  = NULL;
-
-	int ret = -1;
-
-	int i = 0;
-	int32_t call_count = 0;
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t *priv        = NULL;
+        int            child_count = 0;
+        xlator_list_t *trav        = NULL;
+        int            i           = 0;
+        int            ret         = -1;
+        GF_UNUSED int  op_errno    = 0;
+        xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
+        xlator_t      *fav_child   = NULL;
+        char          *qtype       = NULL;
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "replicate translator needs more than one "
+                        "subvolume defined.");
+                return -1;
+        }
 
-	priv = this->private;
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Volume is dangling.");
+        }
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+	this->private = GF_CALLOC (1, sizeof (afr_private_t),
+                                   gf_afr_mt_afr_private_t);
+        if (!this->private)
+                goto out;
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        priv = this->private;
+        LOCK_INIT (&priv->lock);
+        LOCK_INIT (&priv->read_child_lock);
+        //lock recovery is not done in afr
+        pthread_mutex_init (&priv->mutex, NULL);
+        INIT_LIST_HEAD (&priv->saved_fds);
 
-	call_count = local->call_count;
-	frame->local = local;
+        child_count = xlator_subvolume_count (this);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_fentrylk_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->fentrylk,
-				    fd, basename, cmd, type);
+        priv->child_count = child_count;
 
-			if (!--call_count)
-				break;
-		}
-	}
+        priv->read_child = -1;
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
-}
-
-
-int32_t
-afr_checksum_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  uint8_t *file_checksum, uint8_t *dir_checksum)
-		
-{
-	afr_local_t *local = NULL;
-	
-	int call_count = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0 && (local->op_ret != 0)) {
-			local->op_ret = 0;
-
-			local->cont.checksum.file_checksum = MALLOC (ZR_FILENAME_MAX);
-			memcpy (local->cont.checksum.file_checksum, file_checksum, 
-				ZR_FILENAME_MAX);
+        GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
+        if (read_subvol) {
+                priv->read_child = xlator_subvolume_index (this, read_subvol);
+                if (priv->read_child == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
+                                read_subvol->name);
+                        goto out;
+                }
+        }
+        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
+        if (read_subvol_index > -1) {
+                if (read_subvol_index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                read_subvol_index);
+                        goto out;
+                }
+                priv->read_child = read_subvol_index;
+        }
+        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
+
+        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
+
+        priv->favorite_child = -1;
+        GF_OPTION_INIT ("favorite-child", fav_child, xlator, out);
+        if (fav_child) {
+                priv->favorite_child = xlator_subvolume_index (this, fav_child);
+                if (priv->favorite_child == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
+                                fav_child->name);
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_WARNING,
+                        favorite_child_warning_str, fav_child->name,
+                        fav_child->name, fav_child->name);
+        }
 
-			local->cont.checksum.dir_checksum = MALLOC (ZR_FILENAME_MAX);
-			memcpy (local->cont.checksum.dir_checksum, dir_checksum, 
-				ZR_FILENAME_MAX);
 
-		}
+        GF_OPTION_INIT ("background-self-heal-count",
+                        priv->background_self_heal_count, uint32, out);
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        GF_OPTION_INIT ("data-self-heal", priv->data_self_heal, str, out);
 
-	call_count = afr_frame_return (frame);
+        GF_OPTION_INIT ("data-self-heal-algorithm",
+                        priv->data_self_heal_algorithm, str, out);
 
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  local->cont.checksum.file_checksum, 
-				  local->cont.checksum.dir_checksum);
+        GF_OPTION_INIT ("data-self-heal-window-size",
+                        priv->data_self_heal_window_size, uint32, out);
 
-	return 0;
-}
+        GF_OPTION_INIT ("metadata-self-heal", priv->metadata_self_heal, bool,
+                        out);
 
+        GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out);
 
-int32_t
-afr_checksum (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	      int32_t flag)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local  = NULL;
+        GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
 
-	int ret = -1;
+        GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
 
-	int i = 0;
-	int32_t call_count = 0;
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
+        GF_OPTION_INIT ("data-change-log", priv->data_change_log, bool, out);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        GF_OPTION_INIT ("metadata-change-log", priv->metadata_change_log, bool,
+                        out);
 
-	priv = this->private;
+        GF_OPTION_INIT ("entry-change-log", priv->entry_change_log, bool, out);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        GF_OPTION_INIT ("optimistic-change-log", priv->optimistic_change_log,
+                        bool, out);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        GF_OPTION_INIT ("inodelk-trace", priv->inodelk_trace, bool, out);
 
-	call_count = local->call_count;
-	frame->local = local;
+        GF_OPTION_INIT ("entrylk-trace", priv->entrylk_trace, bool, out);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_checksum_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->checksum,
-				    loc, flag);
+        GF_OPTION_INIT ("strict-readdir", priv->strict_readdir, bool, out);
 
-			if (!--call_count)
-				break;
-		}
-	}
+        GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
+        GF_OPTION_INIT ("quorum-type", qtype, str, out);
+        GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
+        GF_OPTION_INIT (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size,
+                        out);
+        fix_quorum_options(this,priv,qtype);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	return 0;
-}
+	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+        GF_OPTION_INIT ("readdir-failover", priv->readdir_failover, bool, out);
+        GF_OPTION_INIT ("ensure-durability", priv->ensure_durability, bool,
+                        out);
 
+        priv->wait_count = 1;
 
-int32_t
-afr_statfs_cbk (call_frame_t *frame, void *cookie,
-		xlator_t *this, int32_t op_ret, int32_t op_errno,
-		struct statvfs *statvfs)
-{
-	afr_local_t *local = NULL;
+        priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
+                                    gf_afr_mt_char);
+        if (!priv->child_up) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-	int call_count = 0;
+        for (i = 0; i < child_count; i++)
+                priv->child_up[i] = -1; /* start with unknown state.
+                                           this initialization needed
+                                           for afr_notify() to work
+                                           reliably
+                                        */
+
+        priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
+                                    gf_afr_mt_xlator_t);
+        if (!priv->children) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-	LOCK (&frame->lock);
-	{
-		local = frame->local;
+        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
+                                       child_count,
+                                       gf_afr_mt_char);
+        if (!priv->pending_key) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-		if (op_ret == 0) {
-			local->op_ret   = op_ret;
-			
-			if (local->cont.statfs.buf_set) {
-				if (statvfs->f_bavail < local->cont.statfs.buf.f_bavail)
-					local->cont.statfs.buf = *statvfs;
-			} else {
-				local->cont.statfs.buf = *statvfs;
-				local->cont.statfs.buf_set = 1;
-			}
-		}
+        trav = this->children;
+        i = 0;
+        while (i < child_count) {
+                priv->children[i] = trav->xlator;
+
+                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
+                                   AFR_XATTR_PREFIX,
+                                   trav->xlator->name);
+                if (-1 == ret) {
+                        ret = -ENOMEM;
+                        goto out;
+                }
 
-		if (op_ret == -1)
-			local->op_errno = op_errno;
+                trav = trav->next;
+                i++;
+        }
 
-	}
-	UNLOCK (&frame->lock);
+        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
+                           this->name);
+        if (-1 == ret) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-	call_count = afr_frame_return (frame);
+        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
+                                      gf_afr_mt_int32_t);
+        if (!priv->last_event) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				  &local->cont.statfs.buf);
+        /* keep more local here as we may need them for self-heal etc */
+        this->local_pool = mem_pool_new (afr_local_t, 512);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
 
-	return 0;
-}
+        priv->first_lookup = 1;
+        priv->root_inode = NULL;
 
+        if (!priv->shd.iamshd) {
+                ret = 0;
+                goto out;
+        }
 
-int32_t
-afr_statfs (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc)
-{
-	afr_private_t *  priv        = NULL;
-	int              child_count = 0;
-	afr_local_t   *  local       = NULL;
-	int              i           = 0;
-
-	int ret = -1;
-	int              call_count = 0;
-	int32_t          op_ret      = -1;
-	int32_t          op_errno    = 0;
-
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-	VALIDATE_OR_GOTO (loc, out);
-
-	priv = this->private;
-	child_count = priv->child_count;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	frame->local = local;
-	call_count = local->call_count;
-
-	for (i = 0; i < child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_statfs_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->statfs, 
-				    loc);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	op_ret = 0;
+        ret = -ENOMEM;
+        priv->shd.pos = GF_CALLOC (sizeof (*priv->shd.pos), child_count,
+                                   gf_afr_mt_brick_pos_t);
+        if (!priv->shd.pos)
+                goto out;
+
+        priv->shd.pending = GF_CALLOC (sizeof (*priv->shd.pending), child_count,
+                                       gf_afr_mt_int32_t);
+        if (!priv->shd.pending)
+                goto out;
+
+        priv->shd.inprogress = GF_CALLOC (sizeof (*priv->shd.inprogress),
+                                          child_count, gf_afr_mt_shd_bool_t);
+        if (!priv->shd.inprogress)
+                goto out;
+        priv->shd.timer = GF_CALLOC (sizeof (*priv->shd.timer), child_count,
+                                     gf_afr_mt_shd_timer_t);
+        if (!priv->shd.timer)
+                goto out;
+
+        priv->shd.healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false,
+                                   _destroy_shd_event_data);
+        if (!priv->shd.healed)
+                goto out;
+
+        priv->shd.heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false,
+                                        _destroy_shd_event_data);
+        if (!priv->shd.heal_failed)
+                goto out;
+
+        priv->shd.split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+                                        _destroy_shd_event_data);
+        if (!priv->shd.split_brain)
+                goto out;
+
+        this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
+        if (!this->itable)
+                goto out;
+        priv->root_inode = inode_ref (this->itable->root);
+        GF_OPTION_INIT ("node-uuid", priv->shd.node_uuid, str, out);
+        GF_OPTION_INIT ("heal-timeout", priv->shd.timeout, int32, out);
+        ret = afr_initialise_statistics (this);
+        if (ret)
+                goto out;
+        ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
-	return 0;
-}
-
-
-int32_t
-afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		   int32_t op_ret, int32_t op_errno, struct flock *lock)
-{
-	afr_local_t * local = NULL;
-
-	int call_count = -1;
-
-	local = frame->local;
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  lock);
-
-	return 0;
-}
-
-
-int32_t 
-afr_lk_unlock (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t   * local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int i;
-	int call_count = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_locked_nodes_count (local->cont.lk.locked_nodes, 
-					     priv->child_count);
-
-	if (call_count == 0) {
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  &local->cont.lk.flock);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	local->cont.lk.flock.l_type = F_UNLCK;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->cont.lk.locked_nodes[i]) {
-			STACK_WIND (frame, afr_lk_unlock_cbk,
-				    priv->children[i],
-				    priv->children[i]->fops->lk,
-				    local->fd, F_SETLK, 
-				    &local->cont.lk.flock);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-	    int32_t op_ret, int32_t op_errno, struct flock *lock)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count  = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv  = this->private;
-
-	child_index = (long) cookie;
-
-	call_count = --local->call_count;
-
-	if (!child_went_down (op_ret, op_errno) && (op_ret == -1)) {
-		local->op_ret   = -1;
-		local->op_errno = op_errno;
-
-		afr_lk_unlock (frame, this);
-		return 0;
-	}
-
-	if (op_ret == 0) {
-		local->op_ret        = 0;
-		local->op_errno      = 0;
-		local->cont.lk.flock = *lock;
-		local->cont.lk.locked_nodes[child_index] = 1;
-	}
-
-	child_index++;
-
-	if (child_index < priv->child_count) {
-		STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) child_index,
-				   priv->children[child_index],
-				   priv->children[child_index]->fops->lk,
-				   local->fd, local->cont.lk.cmd, 
-				   &local->cont.lk.flock);
-	} else if (local->op_ret == -1) {
-		/* all nodes have gone down */
-		
-		AFR_STACK_UNWIND (frame, -1, ENOTCONN, &local->cont.lk.flock);
-	} else {
-		/* locking has succeeded on all nodes that are up */
-		
-		AFR_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			      &local->cont.lk.flock);
-	}
-
-	return 0;
+        return ret;
 }
 
 
 int
-afr_lk (call_frame_t *frame, xlator_t *this,
-	fd_t *fd, int32_t cmd,
-	struct flock *flock)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	int i = 0;
-
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	AFR_LOCAL_INIT (local, priv);
-
-	frame->local  = local;
-
-	local->cont.lk.locked_nodes = CALLOC (priv->child_count, 
-					      sizeof (*local->cont.lk.locked_nodes));
-	
-	if (!local->cont.lk.locked_nodes) {
-		gf_log (this->name, GF_LOG_ERROR, "out of memory :(");
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-	local->fd            = fd_ref (fd);
-	local->cont.lk.cmd   = cmd;
-	local->cont.lk.flock = *flock;
-
-	STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
-			   priv->children[i],
-			   priv->children[i]->fops->lk,
-			   fd, cmd, flock);
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (frame, op_ret, op_errno, NULL);
-	}
-	return 0;
-}
-
-
-/**
- * find_child_index - find the child's index in the array of subvolumes
- * @this: AFR
- * @child: child
- */
-
-static int
-find_child_index (xlator_t *this, xlator_t *child)
-{
-	afr_private_t *priv = NULL;
-
-	int i = -1;
-
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if ((xlator_t *) child == priv->children[i])
-			break;
-	}
-
-	return i;
-}
-
-
-int32_t
-notify (xlator_t *this, int32_t event,
-	void *data, ...)
-{
-	afr_private_t *     priv     = NULL;
-	unsigned char *     child_up = NULL;
-
-	int i           = -1;
-	int up_children = 0;
-
-	priv = this->private;
-
-	if (!priv)
-		return 0;
-
-	child_up = priv->child_up;
-
-	switch (event) {
-	case GF_EVENT_CHILD_UP:
-		i = find_child_index (this, data);
-
-		child_up[i] = 1;
-
-		/* 
-		   if all the children were down, and one child came up, 
-		   send notify to parent
-		*/
-
-		for (i = 0; i < priv->child_count; i++)
-			if (child_up[i])
-				up_children++;
-
-		if (up_children == 1)
-			default_notify (this, event, data);
-
-		break;
-
-	case GF_EVENT_CHILD_DOWN:
-		i = find_child_index (this, data);
-
-		child_up[i] = 0;
-		
-		/* 
-		   if all children are down, and this was the last to go down,
-		   send notify to parent
-		*/
-
-		for (i = 0; i < priv->child_count; i++)
-			if (child_up[i])
-				up_children++;
-
-		if (up_children == 0)
-			default_notify (this, event, data);
-
-		break;
-
-	default:
-		default_notify (this, event, data);
-	}
-
-	return 0;
-}
-
-
-static const char *favorite_child_warning_str = "You have specified subvolume '%s' "
-	"as the 'favorite child'. This means that if a discrepancy in the content "
-	"or attributes (ownership, permission, etc.) of a file is detected among "
-	"the subvolumes, the file on '%s' will be considered the definitive "
-	"version and its contents will OVERWRITE the contents of the file on other "
-	"subvolumes. All versions of the file except that on '%s' "
-	"WILL BE LOST.";
-
-static const char *no_lock_servers_warning_str = "You have set lock-server-count = 0. "
-	"This means correctness is NO LONGER GUARANTEED in all cases. If two or more "
-	"applications write to the same region of a file, there is a possibility that "
-	"its copies will be INCONSISTENT. Set it to a value greater than 0 unless you "
-	"are ABSOLUTELY SURE of what you are doing and WILL NOT HOLD GlusterFS "
-	"RESPOSIBLE for inconsistent data. If you are in doubt, set it to a value "
-	"greater than 0.";
-
-int32_t 
-init (xlator_t *this)
+fini (xlator_t *this)
 {
-	afr_private_t * priv        = NULL;
-	int             child_count = 0;
-	xlator_list_t * trav        = NULL;
-	int             i           = 0;
-	int             ret         = -1;
-	int             op_errno    = 0;
-
-	char * read_subvol = NULL;
-	char * fav_child   = NULL;
-	char * self_heal   = NULL;
-	char * change_log  = NULL;
-
-	int32_t lock_server_count = 1;
-
-	int    fav_ret       = -1;
-	int    read_ret      = -1;
-	int    dict_ret      = -1;
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"AFR needs more than one child defined");
-		return -1;
-	}
-  
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	ALLOC_OR_GOTO (this->private, afr_private_t, out);
-
-	priv = this->private;
-
-	read_ret = dict_get_str (this->options, "read-subvolume", &read_subvol);
-	priv->read_child = -1;
-
-	fav_ret = dict_get_str (this->options, "favorite-child", &fav_child);
-	priv->favorite_child = -1;
-
-	/* Default values */
-
-	priv->data_self_heal     = 1;
-	priv->metadata_self_heal = 1;
-	priv->entry_self_heal    = 1;
-
-	dict_ret = dict_get_str (this->options, "data-self-heal", &self_heal);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (self_heal, &priv->data_self_heal);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"invalid 'option data-self-heal %s' "
-				"defaulting to data-self-heal as 'on'",
-				self_heal);
-			priv->data_self_heal = 1;
-		} 
-	}
-
-	dict_ret = dict_get_str (this->options, "metadata-self-heal",
-				 &self_heal);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (self_heal, &priv->metadata_self_heal);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"invalid 'option metadata-self-heal %s' "
-				"defaulting to metadata-self-heal as 'on'", 
-				self_heal);
-			priv->metadata_self_heal = 1;
-		} 
-	}
-
-	dict_ret = dict_get_str (this->options, "entry-self-heal", &self_heal);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (self_heal, &priv->entry_self_heal);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"invalid 'option entry-self-heal %s' "
-				"defaulting to entry-self-heal as 'on'", 
-				self_heal);
-			priv->entry_self_heal = 1;
-		} 
-	}
-
-	/* Change log options */
-
-	priv->data_change_log     = 1;
-	priv->metadata_change_log = 0;
-	priv->entry_change_log    = 1;
-
-	dict_ret = dict_get_str (this->options, "data-change-log",
-				 &change_log);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (change_log, &priv->data_change_log);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"invalid 'option data-change-log %s'. "
-				"defaulting to data-change-log as 'on'", 
-				change_log);
-			priv->data_change_log = 1;
-		} 
-	}
-
-	dict_ret = dict_get_str (this->options, "metadata-change-log",
-				 &change_log);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (change_log,
-					 &priv->metadata_change_log);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"invalid 'option metadata-change-log %s'. "
-				"defaulting to metadata-change-log as 'off'",
-				change_log);
-			priv->metadata_change_log = 0;
-		} 
-	}
-
-	dict_ret = dict_get_str (this->options, "entry-change-log",
-				 &change_log);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (change_log, &priv->entry_change_log);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"invalid 'option entry-change-log %s'. "
-				"defaulting to entry-change-log as 'on'", 
-				change_log);
-			priv->entry_change_log = 1;
-		} 
-	}
-
-	/* Locking options */
-
-	priv->data_lock_server_count = 1;
-	priv->metadata_lock_server_count = 0;
-	priv->entry_lock_server_count = 1;
-
-	dict_ret = dict_get_int32 (this->options, "data-lock-server-count", 
-				   &lock_server_count);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"setting data lock server count to %d",
-			lock_server_count);
-
-		if (lock_server_count == 0) 
-			gf_log (this->name, GF_LOG_WARNING,
-				no_lock_servers_warning_str);
-
-		priv->data_lock_server_count = lock_server_count;
-	}
-
-
-	dict_ret = dict_get_int32 (this->options,
-				   "metadata-lock-server-count", 
-				   &lock_server_count);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"setting metadata lock server count to %d",
-			lock_server_count);
-		priv->metadata_lock_server_count = lock_server_count;
-	}
-
-
-	dict_ret = dict_get_int32 (this->options, "entry-lock-server-count", 
-				   &lock_server_count);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"setting entry lock server count to %d",
-			lock_server_count);
-
-		priv->entry_lock_server_count = lock_server_count;
-	}
-
-
-	trav = this->children;
-	while (trav) {
-		if (!read_ret && !strcmp (read_subvol, trav->xlator->name)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"subvolume '%s' specified as read child",
-				trav->xlator->name);
-
-			priv->read_child = child_count;
-		}
-
-		if (fav_ret == 0 && !strcmp (fav_child, trav->xlator->name)) {
-			gf_log (this->name, GF_LOG_WARNING,
-				favorite_child_warning_str, trav->xlator->name,
-				trav->xlator->name, trav->xlator->name);
-			priv->favorite_child = child_count;
-		}
-
-		child_count++;
-		trav = trav->next;
-	}
-
-	/* XXX: return inode numbers from 1st subvolume till
-	   afr supports read-subvolume based on inode's ctx 
-	   (and not itransform) for this reason afr_deitransform() 
-	   returns 0 always
-	*/
-	priv->read_child = 0;
-
-	priv->wait_count = 1;
-
-	priv->child_count = child_count;
-	LOCK_INIT (&priv->lock);
-
-	priv->child_up = CALLOC (sizeof (unsigned char), child_count);
-	if (!priv->child_up) {
-		gf_log (this->name, GF_LOG_ERROR,	
-			"out of memory :(");		
-		op_errno = ENOMEM;			
-		goto out;
-	}
-
-	priv->children = CALLOC (sizeof (xlator_t *), child_count);
-	if (!priv->children) {
-		gf_log (this->name, GF_LOG_ERROR,	
-			"out of memory :(");		
-		op_errno = ENOMEM;			
-		goto out;
-	}
-
-	trav = this->children;
-	i = 0;
-	while (i < child_count) {
-		priv->children[i] = trav->xlator;
-
-		trav = trav->next;
-		i++;
-	}
-
-	ret = 0;
-out:
-	return ret;
-}
+        afr_private_t *priv = NULL;
 
+        priv = this->private;
+        this->private = NULL;
+        afr_priv_destroy (priv);
+        if (this->itable);//I dont see any destroy func
 
-int
-fini (xlator_t *this)
-{
-	return 0;
+        return 0;
 }
 
 
 struct xlator_fops fops = {
-	.lookup      = afr_lookup,
-	.open        = afr_open,
-	.lk          = afr_lk,
-	.flush       = afr_flush,
-	.statfs      = afr_statfs,
-	.fsync       = afr_fsync,
-	.fsyncdir    = afr_fsyncdir,
-	.xattrop     = afr_xattrop,
-	.fxattrop    = afr_fxattrop,
-	.inodelk     = afr_inodelk,
-	.finodelk    = afr_finodelk,
-	.entrylk     = afr_entrylk,
-	.fentrylk    = afr_fentrylk,
-	.checksum    = afr_checksum,
-
-	/* inode read */
-	.access      = afr_access,
-	.stat        = afr_stat,
-	.fstat       = afr_fstat,
-	.readlink    = afr_readlink,
-	.getxattr    = afr_getxattr,
-	.readv       = afr_readv,
-
-	/* inode write */
-	.chmod       = afr_chmod,
-	.chown       = afr_chown,
-	.fchmod      = afr_fchmod,
-	.fchown      = afr_fchown,
-	.writev      = afr_writev,
-	.truncate    = afr_truncate,
-	.ftruncate   = afr_ftruncate,
-	.utimens     = afr_utimens,
-	.setxattr    = afr_setxattr,
-	.removexattr = afr_removexattr,
-
-	/* dir read */
-	.opendir     = afr_opendir,
-	.readdir     = afr_readdir,
-	.getdents    = afr_getdents,
-
-	/* dir write */
-	.create      = afr_create,
-	.mknod       = afr_mknod,
-	.mkdir       = afr_mkdir,
-	.unlink      = afr_unlink,
-	.rmdir       = afr_rmdir,
-	.link        = afr_link,
-	.symlink     = afr_symlink,
-	.rename      = afr_rename,
-	.setdents    = afr_setdents,
+        .lookup      = afr_lookup,
+        .open        = afr_open,
+        .lk          = afr_lk,
+        .flush       = afr_flush,
+        .statfs      = afr_statfs,
+        .fsync       = afr_fsync,
+        .fsyncdir    = afr_fsyncdir,
+        .xattrop     = afr_xattrop,
+        .fxattrop    = afr_fxattrop,
+        .inodelk     = afr_inodelk,
+        .finodelk    = afr_finodelk,
+        .entrylk     = afr_entrylk,
+        .fentrylk    = afr_fentrylk,
+	.fallocate   = afr_fallocate,
+	.discard     = afr_discard,
+        .zerofill    = afr_zerofill,
+
+        /* inode read */
+        .access      = afr_access,
+        .stat        = afr_stat,
+        .fstat       = afr_fstat,
+        .readlink    = afr_readlink,
+        .getxattr    = afr_getxattr,
+        .fgetxattr   = afr_fgetxattr,
+        .readv       = afr_readv,
+
+        /* inode write */
+        .writev      = afr_writev,
+        .truncate    = afr_truncate,
+        .ftruncate   = afr_ftruncate,
+        .setxattr    = afr_setxattr,
+        .fsetxattr   = afr_fsetxattr,
+        .setattr     = afr_setattr,
+        .fsetattr    = afr_fsetattr,
+        .removexattr = afr_removexattr,
+        .fremovexattr = afr_fremovexattr,
+
+        /* dir read */
+        .opendir     = afr_opendir,
+        .readdir     = afr_readdir,
+        .readdirp    = afr_readdirp,
+
+        /* dir write */
+        .create      = afr_create,
+        .mknod       = afr_mknod,
+        .mkdir       = afr_mkdir,
+        .unlink      = afr_unlink,
+        .rmdir       = afr_rmdir,
+        .link        = afr_link,
+        .symlink     = afr_symlink,
+        .rename      = afr_rename,
 };
 
 
-struct xlator_mops mops = {
+struct xlator_dumpops dumpops = {
+        .priv       = afr_priv_dump,
 };
 
 
 struct xlator_cbks cbks = {
+        .release     = afr_release,
+        .releasedir  = afr_releasedir,
+        .forget      = afr_forget,
 };
 
+
 struct volume_options options[] = {
-	{ .key  = {"read-subvolume" }, 
-	  .type = GF_OPTION_TYPE_XLATOR
-	},
-	{ .key  = {"favorite-child"}, 
-	  .type = GF_OPTION_TYPE_XLATOR
-	},
-	{ .key  = {"data-self-heal"},  
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = {"metadata-self-heal"},  
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {"entry-self-heal"},  
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = {"data-change-log"},  
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = {"metadata-change-log"},  
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {"entry-change-log"},  
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {"data-lock-server-count"},  
-	  .type = GF_OPTION_TYPE_INT, 
-	  .min  = 0
-	},
-	{ .key  = {"metadata-lock-server-count"},  
-	  .type = GF_OPTION_TYPE_INT, 
-	  .min  = 0
-	},
-	{ .key  = {"entry-lock-server-count"},  
-	  .type = GF_OPTION_TYPE_INT,
-	  .min  = 0
-	},
-	{ .key  = {NULL} },
+        { .key  = {"read-subvolume" },
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. Afr will prefer the one specified using "
+                         "this option if it is not stale. Option value must be "
+                         "one of the xlator names of the children. "
+                         "Ex: <volname>-client-0 till "
+                         "<volname>-client-<number-of-bricks - 1>"
+        },
+        { .key  = {"read-subvolume-index" },
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "-1",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one specified using "
+                         "this option if it is not stale. allowed options"
+                         " include -1 till replica-count - 1"
+        },
+        { .key = {"read-hash-mode" },
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .max = 2,
+          .default_value = "0",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one computed using "
+                         "the method specified using this option"
+                         "0 = first responder, "
+                         "1 = hash by GFID of file (all clients use "
+                                                    "same subvolume), "
+                         "2 = hash by GFID of file and client PID",
+        },
+        { .key  = {"choose-local" },
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "true",
+          .description = "Choose a local subvolume(i.e. Brick) to read from if "
+                         "read-subvolume is not explicitly set.",
+        },
+        { .key  = {"favorite-child"},
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "If a split-brain happens choose subvol/brick set by "
+                         "this option as source."
+        },
+        { .key  = {"background-self-heal-count"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 0,
+          .default_value = "16",
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "This specifies the number of self-heals that can be "
+                         " performed in background without blocking the fop"
+        },
+        { .key  = {"data-self-heal"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = {"1", "on", "yes", "true", "enable",
+                    "0", "off", "no", "false", "disable",
+                    "open"},
+          .default_value = "on",
+          .description   = "Using this option we can enable/disable data "
+                           "self-heal on the file. \"open\" means data "
+                           "self-heal action will only be triggered by file "
+                           "open operations."
+        },
+        { .key  = {"data-self-heal-algorithm"},
+          .type = GF_OPTION_TYPE_STR,
+          .description   = "Select between \"full\", \"diff\". The "
+                           "\"full\" algorithm copies the entire file from "
+                           "source to sink. The \"diff\" algorithm copies to "
+                           "sink only those blocks whose checksums don't match "
+                           "with those of source. If no option is configured "
+                           "the option is chosen dynamically as follows: "
+                           "If the file does not exist on one of the sinks "
+                           "or empty file exists or if the source file size is "
+                           "about the same as page size the entire file will "
+                           "be read and written i.e \"full\" algo, "
+                           "otherwise \"diff\" algo is chosen.",
+          .value = { "diff", "full"}
+        },
+        { .key  = {"data-self-heal-window-size"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .max  = 1024,
+          .default_value = "1",
+          .description = "Maximum number blocks per file for which self-heal "
+                         "process would be applied simultaneously."
+        },
+        { .key  = {"metadata-self-heal"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Using this option we can enable/disable metadata "
+                         "i.e. Permissions, ownerships, xattrs self-heal on "
+                         "the file/directory."
+        },
+        { .key  = {"entry-self-heal"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Using this option we can enable/disable entry "
+                         "self-heal on the directory."
+        },
+        { .key  = {"data-change-log"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Data fops like write/truncate will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
+        },
+        { .key  = {"metadata-change-log"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Metadata fops like setattr/setxattr will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
+        },
+        { .key  = {"entry-change-log"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Entry fops like create/unlink will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
+        },
+        { .key  = {"optimistic-change-log"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Entry/Metadata fops will not perform "
+                         "pre fop changelog operations in afr transaction "
+                         "if this option is enabled."
+        },
+        { .key  = {"strict-readdir"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+        { .key = {"inodelk-trace"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Enabling this option logs inode lock/unlocks"
+        },
+        { .key = {"entrylk-trace"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Enabling this option logs entry lock/unlocks"
+        },
+        { .key = {"eager-lock"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Lock phase of a transaction has two sub-phases. "
+                         "First is an attempt to acquire locks in parallel by "
+                         "broadcasting non-blocking lock requests. If lock "
+                         "aquistion fails on any server, then the held locks "
+                         "are unlocked and revert to a blocking locked mode "
+                         "sequentially on one server after another.  If this "
+                         "option is enabled the initial broadcasting lock "
+                         "request attempt to acquire lock on the entire file. "
+                         "If this fails, we revert back to the sequential "
+                         "\"regional\" blocking lock as before. In the case "
+                         "where such an \"eager\" lock is granted in the "
+                         "non-blocking phase, it gives rise to an opportunity "
+                         "for optimization. i.e, if the next write transaction "
+                         "on the same FD arrives before the unlock phase of "
+                         "the first transaction, it \"takes over\" the full "
+                         "file lock. Similarly if yet another data transaction "
+                         "arrives before the unlock phase of the \"optimized\" "
+                         "transaction, that in turn \"takes over\" the lock as "
+                         "well. The actual unlock now happens at the end of "
+                         "the last \"optimzed\" transaction."
+
+        },
+        { .key = {"self-heal-daemon"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option applies to only self-heal-daemon. "
+                         "Index directory crawl and automatic healing of files"
+                         "will not be performed if this option is turned off."
+        },
+        { .key = {"iam-self-heal-daemon"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option differentiates if the replicate "
+                         "translator is running as part of self-heal-daemon "
+                         "or not."
+        },
+        { .key = {"quorum-type"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = { "none", "auto", "fixed"},
+          .default_value = "none",
+          .description = "If value is \"fixed\" only allow writes if "
+                         "quorum-count bricks are present.  If value is "
+                         "\"auto\" only allow writes if more than half of "
+                         "bricks, or exactly half including the first, are "
+                         "present.",
+        },
+        { .key = {"quorum-count"},
+          .type = GF_OPTION_TYPE_INT,
+          .min = 1,
+          .max = INT_MAX,
+          .default_value = 0,
+          .description = "If quorum-type is \"fixed\" only allow writes if "
+                         "this many bricks or present.  Other quorum types "
+                         "will OVERWRITE this value.",
+        },
+        { .key  = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Local glusterd uuid string, used in starting "
+                         "self-heal-daemon so that it can crawl only on "
+                         "local index directories.",
+        },
+        { .key  = {"heal-timeout"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 60,
+          .max  = INT_MAX,
+          .default_value = "600",
+          .description = "time interval for checking the need to self-heal "
+                         "in self-heal-daemon"
+        },
+        { .key  = {"post-op-delay-secs"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 0,
+          .max  = INT_MAX,
+          .default_value = "1",
+          .description = "Time interval induced artificially before "
+	                 "post-operation phase of the transaction to "
+                         "enhance overlap of adjacent write operations.",
+        },
+        { .key = {AFR_SH_READDIR_SIZE_KEY},
+          .type = GF_OPTION_TYPE_SIZET,
+          .description = "readdirp size for performing entry self-heal",
+          .min = 1024,
+          .max = 131072,
+          .default_value = "1KB",
+        },
+        { .key = {"readdir-failover"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "readdir(p) will not failover if this option is off",
+          .default_value = "on",
+        },
+        { .key = {"ensure-durability"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "Afr performs fsyncs for transactions if this "
+                         "option is on to make sure the changelogs/data is "
+                         "written to the disk",
+          .default_value = "on",
+        },
+        { .key  = {NULL} },
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 4cf6cdf9d..21064db58 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -26,406 +17,919 @@
 #include "config.h"
 #endif
 
-#include "scheduler.h"
 #include "call-stub.h"
 #include "compat-errno.h"
+#include "afr-mem-types.h"
+#include "afr-self-heal-algorithm.h"
+
+#include "libxlator.h"
+#include "timer.h"
+
+#define AFR_XATTR_PREFIX "trusted.afr"
+#define AFR_PATHINFO_HEADER "REPLICATE:"
+#define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
+#define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
+
+#define AFR_LOCKEE_COUNT_MAX    3
+#define AFR_DOM_COUNT_MAX    3
+
+struct _pump_private;
+
+typedef int (*afr_expunge_done_cbk_t) (call_frame_t *frame, xlator_t *this,
+                                       int child, int32_t op_error,
+                                       int32_t op_errno);
+
+typedef int (*afr_impunge_done_cbk_t) (call_frame_t *frame, xlator_t *this,
+                                       int32_t op_error, int32_t op_errno);
+typedef int (*afr_post_remove_call_t) (call_frame_t *frame, xlator_t *this);
+
+typedef int (*afr_lock_cbk_t) (call_frame_t *frame, xlator_t *this);
+typedef void (*afr_lookup_done_cbk_t) (call_frame_t *frame, xlator_t *this,
+                                      int32_t op_ret, int32_t op_errno);
+
+typedef enum {
+        AFR_POS_UNKNOWN,
+        AFR_POS_LOCAL,
+        AFR_POS_REMOTE
+} afr_child_pos_t;
+
+typedef enum {
+        SPLIT_BRAIN = 1,
+        ALL_FOOLS = 2
+} afr_subvol_status_t;
+
+typedef enum {
+        AFR_INODE_SET_READ_CTX = 1,
+        AFR_INODE_RM_STALE_CHILDREN,
+        AFR_INODE_SET_OPENDIR_DONE,
+        AFR_INODE_GET_READ_CTX,
+        AFR_INODE_GET_OPENDIR_DONE,
+} afr_inode_op_t;
+
+typedef struct afr_inode_params_ {
+        afr_inode_op_t op;
+        union {
+                gf_boolean_t value;
+                struct {
+                        int32_t read_child;
+                        int32_t *children;
+                } read_ctx;
+        } u;
+} afr_inode_params_t;
+
+typedef enum afr_spb_state {
+        DONT_KNOW,
+        SPB,
+        NO_SPB
+} afr_spb_state_t;
+
+typedef struct afr_inode_ctx_ {
+        uint64_t masks;
+        int32_t  *fresh_children;//increasing order of latency
+        afr_spb_state_t mdata_spb;
+        afr_spb_state_t data_spb;
+        uint32_t        open_fd_count;
+} afr_inode_ctx_t;
 
+typedef enum {
+        NONE,
+        INDEX,
+        INDEX_TO_BE_HEALED,
+        FULL,
+} afr_crawl_type_t;
+
+typedef struct afr_self_heald_ {
+        gf_boolean_t            enabled;
+        gf_boolean_t            iamshd;
+        afr_crawl_type_t        *pending;
+        gf_boolean_t            *inprogress;
+        afr_child_pos_t         *pos;
+        gf_timer_t              **timer;
+        eh_t                    *healed;
+        eh_t                    *heal_failed;
+        eh_t                    *split_brain;
+        eh_t                    **statistics;
+        void                    **crawl_events;
+        char                    *node_uuid;
+        int                     timeout;
+} afr_self_heald_t;
 
 typedef struct _afr_private {
-	gf_lock_t lock;               /* to guard access to child_count, etc */
-	unsigned int child_count;     /* total number of children   */
+        gf_lock_t lock;               /* to guard access to child_count, etc */
+        unsigned int child_count;     /* total number of children   */
 
-	xlator_t **children;
+        unsigned int read_child_rr;   /* round-robin index of the read_child */
+        gf_lock_t read_child_lock;    /* lock to protect above */
 
-	unsigned char *child_up;
+        xlator_t **children;
 
-	gf_boolean_t data_self_heal;       /* on/off */
-	gf_boolean_t metadata_self_heal;   /* on/off */
-	gf_boolean_t entry_self_heal;      /* on/off */
+        int first_lookup;
+        inode_t *root_inode;
 
+        unsigned char *child_up;
 
-	gf_boolean_t data_change_log;       /* on/off */
-	gf_boolean_t metadata_change_log;   /* on/off */
-	gf_boolean_t entry_change_log;      /* on/off */
+        char **pending_key;
 
-	unsigned int read_child;      /* read-subvolume */
-	unsigned int favorite_child;  /* subvolume to be preferred in resolving
-					 split-brain cases */
+        char         *data_self_heal;              /* on/off/open */
+        char *       data_self_heal_algorithm;    /* name of algorithm */
+        unsigned int data_self_heal_window_size;  /* max number of pipelined
+                                                     read/writes */
 
-	unsigned int data_lock_server_count;
-	unsigned int metadata_lock_server_count;
-	unsigned int entry_lock_server_count;
+        unsigned int background_self_heal_count;
+        unsigned int background_self_heals_started;
+        gf_boolean_t metadata_self_heal;   /* on/off */
+        gf_boolean_t entry_self_heal;      /* on/off */
 
-	unsigned int wait_count;      /* # of servers to wait for success */
-} afr_private_t;
+        gf_boolean_t data_change_log;       /* on/off */
+        gf_boolean_t metadata_change_log;   /* on/off */
+        gf_boolean_t entry_change_log;      /* on/off */
 
-typedef struct {
-	/* array of stat's, one for each child */
-	struct stat *buf;
+        int read_child;               /* read-subvolume */
+        unsigned int hash_mode;       /* for when read_child is not set */
+        int favorite_child;  /* subvolume to be preferred in resolving
+                                         split-brain cases */
 
-	/* array of xattr's, one for each child */
-	dict_t **xattr;
+        gf_boolean_t inodelk_trace;
+        gf_boolean_t entrylk_trace;
 
-	/* array of errno's, one for each child */
-	int *child_errno;
+        gf_boolean_t strict_readdir;
 
-	int32_t **pending_matrix;
-	int32_t **delta_matrix;
+        unsigned int wait_count;      /* # of servers to wait for success */
 
-	int *sources;
-	int source;
-	int active_source;
-	int active_sinks;
-	int *success;
+        uint64_t up_count;      /* number of CHILD_UPs we have seen */
+        uint64_t down_count;    /* number of CHILD_DOWNs we have seen */
 
-	fd_t *healing_fd;
-	int   op_failed;
+        struct _pump_private *pump_private; /* Set if we are loaded as pump */
+        int                   use_afr_in_pump;
 
-	int   file_has_holes;
-	blksize_t block_size;
-	off_t file_size;
-	off_t offset;
+        pthread_mutex_t  mutex;
+        struct list_head saved_fds;   /* list of fds on which locks have succeeded */
+        gf_boolean_t      optimistic_change_log;
+        gf_boolean_t      eager_lock;
+	uint32_t          post_op_delay_secs;
+        unsigned int      quorum_count;
 
-	loc_t parent_loc;
-	int (*completion_cbk) (call_frame_t *frame, xlator_t *this);
-	call_frame_t *sh_frame;
-} afr_self_heal_t;
+        char                   vol_uuid[UUID_SIZE + 1];
+        int32_t                *last_event;
+        afr_self_heald_t       shd;
+        gf_boolean_t           choose_local;
+        gf_boolean_t           did_discovery;
+        gf_boolean_t           readdir_failover;
+        uint64_t               sh_readdir_size;
+        gf_boolean_t           ensure_durability;
+        char                   *sh_domain;
+} afr_private_t;
 
+typedef enum {
+        AFR_SELF_HEAL_NOT_ATTEMPTED,
+        AFR_SELF_HEAL_STARTED,
+        AFR_SELF_HEAL_FAILED,
+        AFR_SELF_HEAL_SYNC_BEGIN,
+} afr_self_heal_status;
+
+typedef struct {
+        afr_self_heal_status gfid_or_missing_entry_self_heal;
+        afr_self_heal_status metadata_self_heal;
+        afr_self_heal_status data_self_heal;
+        afr_self_heal_status entry_self_heal;
+} afr_sh_status_for_all_type;
+
+typedef enum {
+        AFR_SELF_HEAL_ENTRY,
+        AFR_SELF_HEAL_METADATA,
+        AFR_SELF_HEAL_DATA,
+        AFR_SELF_HEAL_GFID_OR_MISSING_ENTRY,
+        AFR_SELF_HEAL_INVALID = -1,
+} afr_self_heal_type;
 
 typedef enum {
-	AFR_DATA_TRANSACTION,          /* truncate, write, ... */
-	AFR_METADATA_TRANSACTION,      /* chmod, chown, ... */
-	AFR_ENTRY_TRANSACTION,         /* create, rmdir, ... */
-	AFR_ENTRY_RENAME_TRANSACTION,  /* rename */
-	AFR_FLUSH_TRANSACTION,         /* flush */
+        AFR_CHECK_ALL,
+        AFR_CHECK_SPECIFIC,
+} afr_sh_fail_check_type;
+
+struct afr_self_heal_ {
+        /* External interface: These are variables (some optional) that
+           are set by whoever has triggered self-heal */
+
+        gf_boolean_t do_data_self_heal;
+        gf_boolean_t do_metadata_self_heal;
+        gf_boolean_t do_entry_self_heal;
+        gf_boolean_t do_gfid_self_heal;
+        gf_boolean_t do_missing_entry_self_heal;
+        gf_boolean_t force_confirm_spb; /* Check for split-brains even when
+                                           self-heal is turned off */
+
+        gf_boolean_t forced_merge;        /* Is this a self-heal triggered to
+                                             forcibly merge the directories? */
+
+        gf_boolean_t background;          /* do self-heal in background
+                                             if possible */
+        ia_type_t type;                   /* st_mode of the entry we're doing
+                                             self-heal on */
+        inode_t   *inode;                 /* inode on which the self-heal is
+                                             performed on */
+        uuid_t  sh_gfid_req;                 /* gfid self-heal needs to be done
+                                             with this gfid if it is not null */
+
+        /* Function to call to unwind. If self-heal is being done in the
+           background, this function will be called as soon as possible. */
+
+        int (*unwind) (call_frame_t *frame, xlator_t *this, int32_t op_ret,
+                       int32_t op_errno, int32_t sh_failed);
+
+        /* End of external interface members */
+
+
+        /* array of stat's, one for each child */
+        struct iatt *buf;
+        struct iatt *parentbufs;
+        struct iatt parentbuf;
+        struct iatt entrybuf;
+
+        afr_expunge_done_cbk_t expunge_done;
+        afr_impunge_done_cbk_t impunge_done;
+
+        /* array of xattr's, one for each child */
+        dict_t **xattr;
+
+        /* array containing if the lookups succeeded in the order of response
+         */
+        int32_t *success_children;
+        int     success_count;
+        /* array containing the fresh children found in the self-heal process */
+        int32_t *fresh_children;
+        /* array containing the fresh children found in the parent lookup */
+        int32_t *fresh_parent_dirs;
+        /* array of errno's, one for each child */
+        int *child_errno;
+        /*loc used for lookup*/
+        loc_t lookup_loc;
+        int32_t lookup_flags;
+        afr_lookup_done_cbk_t lookup_done;
+
+        int32_t **pending_matrix;
+        int32_t **delta_matrix;
+
+        int32_t op_ret;
+        int32_t op_errno;
+
+        int *sources;
+        int source;
+        int active_source;
+        int active_sinks;
+        unsigned char *success;
+        unsigned char *locked_nodes;
+        int lock_count;
+
+        const char *linkname;
+        gf_boolean_t entries_skipped;
+
+        gf_boolean_t actual_sh_started;
+        gf_boolean_t sync_done;
+        gf_boolean_t data_lock_held;
+        gf_boolean_t sh_dom_lock_held;
+        gf_boolean_t eof_reached;
+        fd_t  *healing_fd;
+        int   file_has_holes;
+        blksize_t block_size;
+        off_t file_size;
+        off_t offset;
+        unsigned char *write_needed;
+        uint8_t *checksum;
+        afr_post_remove_call_t post_remove_call;
+
+        char    *data_sh_info;
+        char    *metadata_sh_info;
+
+        loc_t parent_loc;
+        call_frame_t *orig_frame;
+        call_frame_t *old_loop_frame;
+        gf_boolean_t unwound;
+
+        afr_sh_algo_private_t *private;
+        afr_sh_status_for_all_type  afr_all_sh_status;
+        afr_self_heal_type       sh_type_in_action;
+
+        struct afr_sh_algorithm  *algo;
+        afr_lock_cbk_t data_lock_success_handler;
+        afr_lock_cbk_t data_lock_failure_handler;
+	gf_boolean_t data_lock_block;
+        int (*completion_cbk) (call_frame_t *frame, xlator_t *this);
+        int (*sh_data_algo_start) (call_frame_t *frame, xlator_t *this);
+        int (*algo_completion_cbk) (call_frame_t *frame, xlator_t *this);
+        int (*algo_abort_cbk) (call_frame_t *frame, xlator_t *this);
+        void (*gfid_sh_success_cbk) (call_frame_t *sh_frame, xlator_t *this);
+
+        call_frame_t *sh_frame;
+};
+
+typedef struct afr_self_heal_ afr_self_heal_t;
+
+typedef enum {
+        AFR_DATA_TRANSACTION,          /* truncate, write, ... */
+        AFR_METADATA_TRANSACTION,      /* chmod, chown, ... */
+        AFR_ENTRY_TRANSACTION,         /* create, rmdir, ... */
+        AFR_ENTRY_RENAME_TRANSACTION,  /* rename */
 } afr_transaction_type;
 
-typedef struct _afr_local {
-	unsigned int call_count;
-	unsigned int success_count;
-	unsigned int enoent_count;
+typedef enum {
+        AFR_TRANSACTION_LK,
+        AFR_SELFHEAL_LK,
+} transaction_lk_type_t;
 
-	unsigned int need_metadata_self_heal;
-	unsigned int need_entry_self_heal;
-	unsigned int need_data_self_heal;
-	unsigned int govinda_gOvinda;
+typedef enum {
+        AFR_LOCK_OP,
+        AFR_UNLOCK_OP,
+} afr_lock_op_type_t;
 
-	unsigned int reval_child_index;
-	int32_t op_ret;
-	int32_t op_errno;
+typedef enum {
+        AFR_DATA_SELF_HEAL_LK,
+        AFR_METADATA_SELF_HEAL_LK,
+        AFR_ENTRY_SELF_HEAL_LK,
+}selfheal_lk_type_t;
 
-	int32_t *pending_array;
+typedef enum {
+        AFR_INODELK_TRANSACTION,
+        AFR_INODELK_NB_TRANSACTION,
+        AFR_ENTRYLK_TRANSACTION,
+        AFR_ENTRYLK_NB_TRANSACTION,
+        AFR_INODELK_SELFHEAL,
+        AFR_INODELK_NB_SELFHEAL,
+        AFR_ENTRYLK_SELFHEAL,
+        AFR_ENTRYLK_NB_SELFHEAL,
+} afr_lock_call_type_t;
 
-	loc_t loc;
-	loc_t newloc;
+/*
+  xattr format: trusted.afr.volume = [x y z]
+  x - data pending
+  y - metadata pending
+  z - entry pending
+*/
 
-	fd_t *fd;
+static inline int
+afr_index_for_transaction_type (afr_transaction_type type)
+{
+        switch (type) {
 
-	glusterfs_fop_t fop;
+        case AFR_DATA_TRANSACTION:
+                return 0;
 
-	unsigned char *child_up; 
-	int            child_count;
+        case AFR_METADATA_TRANSACTION:
+                return 1;
 
-	int32_t *child_errno;
-	
-	dict_t  *xattr_req;
-	int      open_fd_count;
-	/* 
-	   This struct contains the arguments for the "continuation"
-	   (scheme-like) of fops
-	*/
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+                return 2;
+        }
 
-	int   op;
-	struct {
-		struct {
-			unsigned char buf_set;
-			struct statvfs buf;
-		} statfs;
+        return -1;  /* make gcc happy */
+}
 
-		struct {
-			inode_t *inode;
-			struct stat buf;
-			dict_t *xattr;
-		} lookup;
+typedef struct {
+        loc_t                   loc;
+        char                    *basename;
+        unsigned char           *locked_nodes;
+        int                     locked_count;
 
-		struct {
-			int32_t flags;
-		} open;
+} afr_entry_lockee_t;
 
-		struct {
-			int32_t cmd;
-			struct flock flock;
-			unsigned char *locked_nodes;
-		} lk;
+int
+afr_entry_lockee_cmp (const void *l1, const void *l2);
 
-		struct {
-			uint8_t *file_checksum;
-			uint8_t *dir_checksum;
-		} checksum;
+typedef struct {
+        char    *domain; /* Domain on which inodelk is taken */
+        struct gf_flock flock;
+        unsigned char *locked_nodes;
+        int32_t lock_count;
+} afr_inodelk_t;
 
-		/* inode read */
+typedef struct {
+        loc_t *lk_loc;
 
-		struct {
-			int32_t mask;
-			int last_tried;  /* index of the child we tried previously */
-		} access;
+        int                     lockee_count;
+        afr_entry_lockee_t      lockee[AFR_LOCKEE_COUNT_MAX];
 
-		struct {
-			int last_tried;
-			ino_t ino;
-		} stat;
+        afr_inodelk_t       inodelk[AFR_DOM_COUNT_MAX];
+        const char *lk_basename;
+        const char *lower_basename;
+        const char *higher_basename;
+        char lower_locked;
+        char higher_locked;
 
-		struct {
-			int last_tried;
-			ino_t ino;
-		} fstat;
+        unsigned char *locked_nodes;
+        unsigned char *lower_locked_nodes;
 
-		struct {
-			size_t size;
-			int last_tried;
-		} readlink;
+        selfheal_lk_type_t selfheal_lk_type;
+        transaction_lk_type_t transaction_lk_type;
 
-		struct {
-			const char *name;
-			int last_tried;
-		} getxattr;
+        int32_t lock_count;
+        int32_t entrylk_lock_count;
 
-		struct {
-			size_t size;
-			off_t offset;
-			int last_tried;
-		} readv;
+        uint64_t lock_number;
+        int32_t lk_call_count;
+        int32_t lk_expected_count;
+        int32_t lk_attempted_count;
 
-		/* dir read */
+        int32_t lock_op_ret;
+        int32_t lock_op_errno;
+        afr_lock_cbk_t lock_cbk;
+        char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
+} afr_internal_lock_t;
 
-		struct {
-			int success_count;
-			int32_t op_ret;
-			int32_t op_errno;
-		} opendir;
+typedef struct _afr_locked_fd {
+        fd_t  *fd;
+        struct list_head list;
+} afr_locked_fd_t;
 
-		struct {
-			int32_t op_ret;
-			int32_t op_errno;
-			size_t size;
-			off_t offset;
+struct afr_reply {
+	int	valid;
+	int32_t	op_ret;
+	int32_t	op_errno;
+};
 
-			int last_tried;
-		} readdir;
+typedef struct _afr_local {
+        int     uid;
+        int     gid;
+        unsigned int call_count;
+        unsigned int success_count;
+        unsigned int enoent_count;
+        uint32_t     open_fd_count;
+        gf_boolean_t update_open_fd_count;
 
-		struct {
-			int32_t op_ret;
-			int32_t op_errno;
 
-			size_t size;
-			off_t offset;
-			int32_t flag;
+        unsigned int unhealable;
 
-			int last_tried;
-		} getdents;
+        unsigned int read_child_index;
+        unsigned char read_child_returned;
+        unsigned int first_up_child;
 
-		/* inode write */
+	gf_lkowner_t  saved_lk_owner;
 
-		struct {
-			ino_t ino;
-			mode_t mode;
-			struct stat buf;
-		} chmod;
+        int32_t op_ret;
+        int32_t op_errno;
 
-		struct {
-			ino_t ino;
-			mode_t mode;
-			struct stat buf;
-		} fchmod;
+        int32_t **pending;
 
-		struct {
-			ino_t ino;
-			uid_t uid;
-			gid_t gid;
-			struct stat buf;
-		} chown;
+        loc_t loc;
+        loc_t newloc;
 
-		struct {
-			ino_t ino;
-			uid_t uid;
-			gid_t gid;
-			struct stat buf;
-		} fchown;
-		
-		struct {
-			ino_t ino;
-			struct stat buf;
+        fd_t *fd;
 
-			int32_t op_ret;
+        glusterfs_fop_t fop;
 
-			struct iovec *vector;
-			dict_t *refs;
-			int32_t count;
-			off_t offset;
-		} writev;
+        unsigned char *child_up;
+        int32_t       *fresh_children; //in the order of response
 
-		struct {
-			ino_t ino;
-			off_t offset;
-			struct stat buf;
-		} truncate;
+        int32_t *child_errno;
 
-		struct {
-			ino_t ino;
-			off_t offset;
-			struct stat buf;
-		} ftruncate;
+        dict_t  *xattr_req;
 
-		struct {
-			ino_t ino;
-			struct timespec tv[2];
-			struct stat buf;
-		} utimens;
+        int32_t  inodelk_count;
+        int32_t  entrylk_count;
 
-		struct {
-			dict_t *dict;
-			int32_t flags;
-		} setxattr;
+        afr_internal_lock_t internal_lock;
 
-		struct {
-			const char *name;
-		} removexattr;
+        afr_locked_fd_t *locked_fd;
+        int32_t          source_child;
+        int32_t          lock_recovery_child;
 
-		/* dir write */
-		
-		struct {
-			ino_t ino;
-			fd_t *fd;
-			int32_t flags;
-			mode_t mode;
-			inode_t *inode;
-			struct stat buf;
-		} create;
+        dict_t  *dict;
+        int      optimistic_change_log;
+	gf_boolean_t      delayed_post_op;
 
-		struct {
-			ino_t ino;
-			dev_t dev;
-			mode_t mode;
-			inode_t *inode;
-			struct stat buf;
-		} mknod;
+
+	/* Is the current writev() going to perform a stable write?
+	   i.e, is fd->flags or @flags writev param have O_SYNC or
+	   O_DSYNC?
+	*/
+        gf_boolean_t      stable_write;
+
+        /* This write appended to the file. Nnot necessarily O_APPEND,
+           just means the offset of write was at the end of file.
+        */
+        gf_boolean_t      append_write;
+
+        int allow_sh_for_running_transaction;
+
+
+        /* This struct contains the arguments for the "continuation"
+           (scheme-like) of fops
+        */
+
+        int   op;
+        struct {
+                struct {
+                        unsigned char buf_set;
+                        struct statvfs buf;
+                } statfs;
+
+                struct {
+                        uint32_t parent_entrylk;
+                        uuid_t  gfid_req;
+                        inode_t *inode;
+                        struct iatt buf;
+                        struct iatt postparent;
+                        dict_t **xattrs;
+                        dict_t *xattr;
+                        struct iatt *postparents;
+                        struct iatt *bufs;
+                        int32_t read_child;
+                        int32_t *sources;
+                        int32_t *success_children;
+                        int32_t **pending_matrix;
+                        gf_boolean_t fresh_lookup;
+                        gf_boolean_t possible_spb;
+                } lookup;
+
+                struct {
+                        int32_t flags;
+                } open;
+
+                struct {
+                        int32_t cmd;
+                        struct gf_flock user_flock;
+                        struct gf_flock ret_flock;
+                        unsigned char *locked_nodes;
+                } lk;
+
+                /* inode read */
+
+                struct {
+                        int32_t mask;
+                        int last_index;  /* index of the child we tried previously */
+                } access;
+
+                struct {
+                        int last_index;
+                } stat;
+
+                struct {
+                        int last_index;
+                } fstat;
+
+                struct {
+                        size_t size;
+                        int last_index;
+                } readlink;
+
+                struct {
+                        char *name;
+                        int last_index;
+                        long xattr_len;
+                } getxattr;
+
+                struct {
+                        size_t size;
+                        off_t offset;
+                        int last_index;
+                        uint32_t flags;
+                } readv;
+
+                /* dir read */
+
+                struct {
+                        int success_count;
+                        int32_t op_ret;
+                        int32_t op_errno;
+
+                        uint32_t *checksum;
+                } opendir;
+
+                struct {
+                        int32_t op_ret;
+                        int32_t op_errno;
+                        size_t size;
+                        off_t offset;
+                        dict_t *dict;
+                        gf_boolean_t failed;
+                        int last_index;
+                } readdir;
+                /* inode write */
+
+                struct {
+                        struct iatt prebuf;
+                        struct iatt postbuf;
+                } inode_wfop; //common structure for all inode-write-fops
+
+                struct {
+                        int32_t op_ret;
+
+                        struct iovec *vector;
+                        struct iobref *iobref;
+                        int32_t count;
+                        off_t offset;
+                        uint32_t flags;
+                } writev;
+
+                struct {
+                        off_t offset;
+                } truncate;
+
+                struct {
+                        off_t offset;
+                } ftruncate;
+
+                struct {
+                        struct iatt in_buf;
+                        int32_t valid;
+                } setattr;
+
+                struct {
+                        struct iatt in_buf;
+                        int32_t valid;
+                } fsetattr;
+
+                struct {
+                        dict_t *dict;
+                        int32_t flags;
+                } setxattr;
+
+                struct {
+                        dict_t *dict;
+                        int32_t flags;
+                } fsetxattr;
+
+                struct {
+                        char *name;
+                } removexattr;
+
+                struct {
+                        dict_t *xattr;
+                } xattrop;
+
+                struct {
+                        dict_t *xattr;
+                } fxattrop;
+
+                /* dir write */
+
+                struct {
+                        inode_t *inode;
+                        struct iatt buf;
+                        struct iatt preparent;
+                        struct iatt postparent;
+                        struct iatt prenewparent;
+                        struct iatt postnewparent;
+                } dir_fop; //common structure for all dir fops
+
+                struct {
+                        fd_t *fd;
+                        dict_t *params;
+                        int32_t flags;
+                        mode_t mode;
+                } create;
+
+                struct {
+                        dev_t dev;
+                        mode_t mode;
+                        dict_t *params;
+                } mknod;
+
+                struct {
+                        int32_t mode;
+                        dict_t *params;
+                } mkdir;
+
+                struct {
+                        int flags;
+                } rmdir;
+
+                struct {
+                        dict_t *params;
+                        char *linkpath;
+                } symlink;
 
 		struct {
-			ino_t ino;
 			int32_t mode;
-			inode_t *inode;
-			struct stat buf;
-		} mkdir;
+			off_t offset;
+			size_t len;
+		} fallocate;
 
 		struct {
-			int32_t op_ret;
-			int32_t op_errno;
-		} unlink;
+			off_t offset;
+			size_t len;
+		} discard;
 
-		struct {
-			int32_t op_ret;
-			int32_t op_errno;
-		} rmdir;
+                struct {
+                        off_t offset;
+                        size_t len;
+                        struct iatt prebuf;
+                        struct iatt postbuf;
+                } zerofill;
 
-		struct {
-			ino_t ino;
-			struct stat buf;
-		} rename;
 
-		struct {
-			ino_t ino;
-			inode_t *inode;
-			struct stat buf;
-		} link;
+        } cont;
 
-		struct {
-			ino_t ino;
-			inode_t *inode;
-			struct stat buf;
-			char *linkpath;
-		} symlink;
+        struct {
+                off_t start, len;
 
-		struct {
-			int32_t flags;
-			dir_entry_t *entries;
-			int32_t count;
-		} setdents;
-	} cont;
-	
-	struct {
-		off_t start, len;
+                gf_boolean_t    eager_lock_on;
+                int *eager_lock;
+
+                char *basename;
+                char *new_basename;
+
+                loc_t parent_loc;
+                loc_t new_parent_loc;
+
+                afr_transaction_type type;
 
-		unsigned char *locked_nodes;
-		int lock_count;
+		/* pre-compute the post piggyback status before
+		   entering POST-OP phase
+		*/
+		int              *postop_piggybacked;
 
-		const char *basename;
-		const char *new_basename;
+		/* stub to resume on destruction
+		   of the transaction frame */
+		call_stub_t      *resume_stub;
 
-		char *pending;
+		struct list_head  eager_locked;
 
-		loc_t parent_loc;
-		loc_t new_parent_loc;
+                int32_t         **txn_changelog;//changelog after pre+post ops
+                unsigned char   *pre_op;
 
-		afr_transaction_type type;
+                call_frame_t *main_frame;
 
-		int success_count;
-		int erase_pending;
-		int failure_count;
+                int (*fop) (call_frame_t *frame, xlator_t *this);
 
-		int last_tried;
-		int32_t *child_errno;
+                int (*done) (call_frame_t *frame, xlator_t *this);
 
-		call_frame_t *main_frame;
+                int (*resume) (call_frame_t *frame, xlator_t *this);
 
-		int (*fop) (call_frame_t *frame, xlator_t *this);
+                int (*unwind) (call_frame_t *frame, xlator_t *this);
 
-		int (*done) (call_frame_t *frame, xlator_t *this);
+                /* post-op hook */
+        } transaction;
 
-		int (*resume) (call_frame_t *frame, xlator_t *this);
+        afr_self_heal_t self_heal;
 
-		int (*unwind) (call_frame_t *frame, xlator_t *this);
-	} transaction;
+        struct marker_str     marker;
 
-	afr_self_heal_t self_heal;
+        /* extra data for fops */
+        dict_t         *xdata_req;
+        dict_t         *xdata_rsp;
+
+        mode_t          umask;
+        int             xflag;
+        gf_boolean_t    do_discovery;
+	struct afr_reply *replies;
 } afr_local_t;
 
+typedef enum {
+        AFR_FD_NOT_OPENED,
+        AFR_FD_OPENED,
+        AFR_FD_OPENING
+} afr_fd_open_status_t;
+
+typedef struct {
+        unsigned int *pre_op_done;
+        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
+        unsigned int *pre_op_piggyback;
+
+        unsigned int *lock_piggyback;
+        unsigned int *lock_acquired;
+
+        int flags;
+        uint64_t up_count;   /* number of CHILD_UPs this fd has seen */
+        uint64_t down_count; /* number of CHILD_DOWNs this fd has seen */
+
+        int32_t last_tried;
+
+        int  hit, miss;
+        gf_boolean_t failed_over;
+        struct list_head entries; /* needed for readdir failover */
+
+        unsigned char *locked_on; /* which subvolumes locks have been successful */
+
+	/* used for delayed-post-op optimization */
+	pthread_mutex_t    delay_lock;
+	gf_timer_t        *delay_timer;
+	call_frame_t      *delay_frame;
+        int               call_child;
+
+	/* set if any write on this fd was a non stable write
+	   (i.e, without O_SYNC or O_DSYNC)
+	*/
+	gf_boolean_t      witnessed_unstable_write;
+
+	/* list of frames currently in progress */
+	struct list_head  eager_locked;
+} afr_fd_ctx_t;
+
+
 /* try alloc and if it fails, goto label */
-#define ALLOC_OR_GOTO(var, type, label) do {			\
-		var = CALLOC (sizeof (type), 1);		\
-		if (!var) {					\
-			gf_log (this->name, GF_LOG_ERROR,	\
-				"out of memory :(");		\
-			op_errno = ENOMEM;			\
-			goto label;				\
-		}						\
-	} while (0);
+#define AFR_LOCAL_ALLOC_OR_GOTO(var, label) do {                    \
+                var = mem_get0 (THIS->local_pool);                  \
+                if (!var) {                                         \
+                        gf_log (this->name, GF_LOG_ERROR,           \
+                                "out of memory :(");                \
+                        op_errno = ENOMEM;                          \
+                        goto label;                                 \
+                }                                                   \
+        } while (0);
 
 
 /* did a call fail due to a child failing? */
-#define child_went_down(op_ret, op_errno) (((op_ret) < 0) &&	      \
-					   ((op_errno == ENOTCONN) || \
-					    (op_errno == EBADFD)))
+#define child_went_down(op_ret, op_errno) (((op_ret) < 0) &&            \
+                                           ((op_errno == ENOTCONN) ||   \
+                                            (op_errno == EBADFD)))
+
+#define afr_fop_failed(op_ret, op_errno) ((op_ret) == -1)
 
 /* have we tried all children? */
 #define all_tried(i, count)  ((i) == (count) - 1)
 
+int32_t
+afr_set_dict_gfid (dict_t *dict, uuid_t gfid);
+
+int
+pump_command_reply (call_frame_t *frame, xlator_t *this);
+
+int32_t
+afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
+
+int
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count);
+
 void
-afr_build_parent_loc (loc_t *parent, loc_t *child);
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock);
 
 int
-afr_up_children_count (int child_count, unsigned char *child_up);
+afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
+afr_save_locked_fd (xlator_t *this, fd_t *fd);
+
+int
+afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
+                       unsigned char *locked_nodes);
+
+void
+afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner);
+
+int
+afr_set_lock_number (call_frame_t *frame, xlator_t *this);
+
+
+loc_t *
+lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2);
+
+int32_t
+afr_unlock (call_frame_t *frame, xlator_t *this);
+
+int
+afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this);
 
 int
-afr_first_up_child (afr_private_t *priv);
+afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this);
 
-ino64_t
-afr_itransform (ino64_t ino, int child_count, int child_index);
+int
+afr_blocking_lock (call_frame_t *frame, xlator_t *this);
 
 int
-afr_deitransform (ino64_t ino, int child_count);
+afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
+
+int
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
+                          unsigned int child_count);
+
+int pump_start (call_frame_t *frame, xlator_t *this);
+
+int
+__afr_fd_ctx_set (xlator_t *this, fd_t *fd);
+
+int
+afr_fd_ctx_set (xlator_t *this, fd_t *fd);
+
+int32_t
+afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children);
+
+void
+afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
+                        int32_t *fresh_children);
+
+int
+afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno);
+
+unsigned int
+afr_up_children_count (unsigned char *child_up, unsigned int child_count);
+
+unsigned int
+afr_locked_children_count (unsigned char *children, unsigned int child_count);
+
+unsigned int
+afr_pre_op_done_children_count (unsigned char *pre_op,
+                                unsigned int child_count);
+
+gf_boolean_t
+afr_is_fresh_lookup (loc_t *loc, xlator_t *this);
+
+void
+afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent);
+
+int
+afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
 
 void
 afr_local_cleanup (afr_local_t *local, xlator_t *this);
@@ -433,91 +937,276 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this);
 int
 afr_frame_return (call_frame_t *frame);
 
-#define AFR_STACK_UNWIND(frame, params ...)		\
-	do {						\
-		afr_local_t *__local = NULL;		\
-		xlator_t    *__this = NULL;		\
-		__local = frame->local;			\
-		__this = frame->this;			\
-		frame->local = NULL;                    \
-		STACK_UNWIND (frame, params);		\
-		afr_local_cleanup (__local, __this);	\
-		free (__local);				\
-} while (0);					
-
-#define AFR_STACK_DESTROY(frame)			\
-	do {						\
-		afr_local_t *__local = NULL;		\
-		xlator_t    *__this = NULL;		\
-		__local = frame->local;			\
-		__this = frame->this;			\
-		frame->local = NULL;                    \
-		STACK_DESTROY (frame->root);		\
-		afr_local_cleanup (__local, __this);	\
-		free (__local);				\
-} while (0);					
+gf_boolean_t
+afr_is_split_brain (xlator_t *this, inode_t *inode);
 
+void
+afr_set_split_brain (xlator_t *this, inode_t *inode, afr_spb_state_t mdata_spb,
+                     afr_spb_state_t data_spb);
+
+int
+afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          fd_t *fd, dict_t *xdata);
+
+void
+afr_set_opendir_done (xlator_t *this, inode_t *inode);
+
+gf_boolean_t
+afr_is_opendir_done (xlator_t *this, inode_t *inode);
+
+void
+afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
+
+int
+afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd);
+
+int
+afr_launch_openfd_self_heal (call_frame_t *frame, xlator_t *this, fd_t *fd);
+
+#define AFR_STACK_UNWIND(fop, frame, params ...)                \
+        do {                                                    \
+                afr_local_t *__local = NULL;                    \
+                xlator_t    *__this = NULL;                     \
+                if (frame) {                                    \
+                        __local = frame->local;                 \
+                        __this = frame->this;                   \
+                        frame->local = NULL;                    \
+                }                                               \
+                STACK_UNWIND_STRICT (fop, frame, params);       \
+                if (__local) {                                  \
+                        afr_local_cleanup (__local, __this);    \
+                        mem_put (__local);                      \
+                }                                               \
+        } while (0)
+
+#define AFR_STACK_DESTROY(frame)                                \
+        do {                                                    \
+                afr_local_t *__local = NULL;                    \
+                xlator_t    *__this = NULL;                     \
+                __local = frame->local;                         \
+                __this = frame->this;                           \
+                frame->local = NULL;                            \
+                STACK_DESTROY (frame->root);                    \
+                if (__local) {                                  \
+                        afr_local_cleanup (__local, __this);    \
+                        mem_put (__local);                      \
+                }                                               \
+        } while (0);
+
+#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
 /* allocate and return a string that is the basename of argument */
-static inline char * 
-AFR_BASENAME (const char *str)						
+static inline char *
+AFR_BASENAME (const char *str)
 {
-	char *__tmp_str = NULL;				
-	char *__basename_str = NULL;			
-	__tmp_str = strdup (str);			
-	__basename_str = strdup (basename (__tmp_str));	
-	FREE (__tmp_str);
-	return __basename_str;
+        char *__tmp_str = NULL;
+        char *__basename_str = NULL;
+        __tmp_str = gf_strdup (str);
+        __basename_str = gf_strdup (basename (__tmp_str));
+        GF_FREE (__tmp_str);
+        return __basename_str;
 }
 
-/* initialize local_t */
-static inline int
-AFR_LOCAL_INIT (afr_local_t *local, afr_private_t *priv)
-{
-	local->child_up = CALLOC (sizeof (*local->child_up),
-				  priv->child_count);
-	if (!local->child_up) {
-		return -ENOMEM;
-	}
+int
+afr_transaction_local_init (afr_local_t *local, xlator_t *this);
 
-	memcpy (local->child_up, priv->child_up, 
-		sizeof (*local->child_up) * priv->child_count);
+int32_t
+afr_marker_getxattr (call_frame_t *frame, xlator_t *this,
+                     loc_t *loc, const char *name,afr_local_t *local, afr_private_t *priv );
 
+int32_t *
+afr_children_create (int32_t child_count);
 
-	local->call_count = afr_up_children_count (priv->child_count, local->child_up);
-	if (local->call_count == 0)
-		return -ENOTCONN;
+int
+afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
 
-	local->transaction.erase_pending = 1;
+int
+afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
+                        transaction_lk_type_t lk_type);
 
-	local->op_ret = -1;
-	local->op_errno = EUCLEAN;
+int
+afr_first_up_child (unsigned char *child_up, size_t child_count);
 
-	return 0;
-}
+int
+afr_select_read_child_from_policy (int32_t *fresh_children, int32_t child_count,
+                                   int32_t prev_read_child,
+                                   int32_t config_read_child, int32_t *sources,
+                                   unsigned int hmode, uuid_t gfid);
 
+void
+afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
+                              int32_t *fresh_children, int32_t prev_read_child,
+                              int32_t config_read_child, uuid_t gfid);
+
+int32_t
+afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
+                    int32_t *fresh_children,
+                    int32_t *call_child, int32_t *last_index);
+
+int32_t
+afr_next_call_child (int32_t *fresh_children, unsigned char *child_up,
+                     size_t child_count, int32_t *last_index,
+                     int32_t read_child);
+void
+afr_get_fresh_children (int32_t *success_children, int32_t *sources,
+                        int32_t *children, unsigned int child_count);
+void
+afr_children_add_child (int32_t *children, int32_t child,
+                              int32_t child_count);
+void
+afr_children_rm_child (int32_t *children, int32_t child,
+                             int32_t child_count);
+void
+afr_reset_children (int32_t *children, int32_t child_count);
+int32_t
+afr_most_important_error(int32_t old_errno, int32_t new_errno,
+			 gf_boolean_t eio);
+int
+afr_errno_count (int32_t *children, int *child_errno,
+                 unsigned int child_count, int32_t op_errno);
+int
+afr_get_children_count (int32_t *children, unsigned int child_count);
+gf_boolean_t
+afr_is_child_present (int32_t *success_children, int32_t child_count,
+                      int32_t child);
+void
+afr_update_gfid_from_iatts (uuid_t uuid, struct iatt *bufs,
+                            int32_t *success_children,
+                            unsigned int child_count);
+void
+afr_reset_xattr (dict_t **xattr, unsigned int child_count);
+gf_boolean_t
+afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
+                        unsigned int child_count, const char *path,
+                        const char *xlator_name);
+unsigned int
+afr_gfid_missing_count (const char *xlator_name, int32_t *children,
+                        struct iatt *bufs, unsigned int child_count,
+                        const char *path);
+void
+afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path);
+void
+afr_children_copy (int32_t *dst, int32_t *src, unsigned int child_count);
+afr_transaction_type
+afr_transaction_type_get (ia_type_t ia_type);
+int32_t
+afr_resultant_errno_get (int32_t *children,
+                         int *child_errno, unsigned int child_count);
+void
+afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
+                             int32_t *stale_children);
+void
+afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t background, ia_type_t ia_type, char *reason,
+                      void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
+                                                   xlator_t *this),
+                      int (*unwind) (call_frame_t *frame, xlator_t *this,
+                                     int32_t op_ret, int32_t op_errno,
+                                     int32_t sh_failed));
+void
+afr_fix_open (xlator_t *this, fd_t *fd, size_t need_open_count, int *need_open);
 
-static inline int
-afr_transaction_local_init (afr_local_t *local, afr_private_t *priv)
-{
-	local->child_errno = CALLOC (sizeof (*local->child_errno),
-				     priv->child_count);
-	if (!local->child_errno) {
-		return -ENOMEM;
-	}
+void
+afr_open_fd_fix (fd_t *fd, xlator_t *this);
+int
+afr_set_elem_count_get (unsigned char *elems, int child_count);
 
-	local->pending_array = CALLOC (sizeof (*local->pending_array),
-				       priv->child_count);
-	if (!local->pending_array) {
-		return -ENOMEM;
-	}
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 
-	local->transaction.locked_nodes = CALLOC (sizeof (*local->transaction.locked_nodes),
-						  priv->child_count);
+gf_boolean_t
+afr_open_only_data_self_heal (char *data_self_heal);
 
-	local->transaction.child_errno = CALLOC (sizeof (*local->transaction.child_errno),
-						  priv->child_count);
+gf_boolean_t
+afr_data_self_heal_enabled (char *data_self_heal);
 
-	return 0;
-}
+void
+afr_set_low_priority (call_frame_t *frame);
+int
+afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
+                      int flags);
+
+gf_boolean_t
+afr_have_quorum (char *logname, afr_private_t *priv);
+
+void
+afr_matrix_cleanup (int32_t **pending, unsigned int m);
+
+int32_t**
+afr_matrix_create (unsigned int m, unsigned int n);
+
+gf_boolean_t
+afr_is_errno_set (int *child_errno, int child);
+
+gf_boolean_t
+afr_is_errno_unset (int *child_errno, int child);
+
+gf_boolean_t
+afr_is_fd_fixable (fd_t *fd);
+
+void
+afr_prepare_new_entry_pending_matrix (int32_t **pending,
+                                      gf_boolean_t (*is_pending) (int *, int),
+                                      int *ctx, struct iatt *buf,
+                                      unsigned int child_count);
+void
+afr_xattr_array_destroy (dict_t **xattr, unsigned int child_count);
+/*
+ * Special value indicating we should use the "auto" quorum method instead of
+ * a fixed value (including zero to turn off quorum enforcement).
+ */
+#define AFR_QUORUM_AUTO INT_MAX
+
+/*
+ * Having this as a macro will make debugging a bit weirder, but does reduce
+ * the probability of functions handling this check inconsistently.
+ */
+#define QUORUM_CHECK(_func,_label) do {                                  \
+        if (priv->quorum_count && !afr_have_quorum(this->name,priv)) { \
+                gf_log(this->name,GF_LOG_WARNING,                        \
+                       "failing "#_func" due to lack of quorum");        \
+                op_errno = EROFS;                                        \
+                goto _label;                                             \
+        }                                                                \
+} while (0);
+
+
+#define AFR_SBRAIN_MSG "Failed on %s as split-brain is seen. Returning EIO."
+
+#define AFR_SBRAIN_CHECK_FD(fd, label) do {                              \
+        if (fd->inode && afr_is_split_brain (this, fd->inode)) {        \
+                op_errno = EIO;                                         \
+                gf_log (this->name, GF_LOG_WARNING,                     \
+                        AFR_SBRAIN_MSG ,uuid_utoa (fd->inode->gfid));   \
+                goto label;                                             \
+        }                                                               \
+} while (0)
+
+#define AFR_SBRAIN_CHECK_LOC(loc, label) do {                           \
+        if (loc->inode && afr_is_split_brain (this, loc->inode)) {      \
+                op_errno = EIO;                                         \
+                loc_path (loc, NULL);                                   \
+                gf_log (this->name, GF_LOG_WARNING,                     \
+                        AFR_SBRAIN_MSG , loc->path);                    \
+                goto label;                                             \
+        }                                                               \
+} while (0)
+
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
+
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd);
+
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub);
+
+int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count);
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this);
+
+afr_inode_ctx_t*
+afr_inode_ctx_get (inode_t *inode, xlator_t *this);
 
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
new file mode 100644
index 000000000..a7f72fb30
--- /dev/null
+++ b/xlators/cluster/afr/src/pump.c
@@ -0,0 +1,2641 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <sys/time.h>
+#include <stdlib.h>
+#include <fnmatch.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "afr-common.c"
+#include "defaults.c"
+#include "glusterfs.h"
+
+static uint64_t pump_pid = 0;
+static inline void
+pump_fill_loc_info (loc_t *loc, struct iatt *iatt, struct iatt *parent)
+{
+        afr_update_loc_gfids (loc, iatt, parent);
+        uuid_copy (loc->inode->gfid, iatt->ia_gfid);
+}
+
+static int
+pump_mark_start_pending (xlator_t *this)
+{
+        afr_private_t  *priv      = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        priv      = this->private;
+        pump_priv = priv->pump_private;
+
+        pump_priv->pump_start_pending = 1;
+
+        return 0;
+}
+
+static int
+is_pump_start_pending (xlator_t *this)
+{
+        afr_private_t  *priv      = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        priv      = this->private;
+        pump_priv = priv->pump_private;
+
+        return (pump_priv->pump_start_pending);
+}
+
+static int
+pump_remove_start_pending (xlator_t *this)
+{
+        afr_private_t  *priv      = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        priv      = this->private;
+        pump_priv = priv->pump_private;
+
+        pump_priv->pump_start_pending = 0;
+
+        return 0;
+}
+
+static pump_state_t
+pump_get_state ()
+{
+        xlator_t *this = NULL;
+        afr_private_t *priv = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        pump_state_t ret;
+
+        this = THIS;
+        priv = this->private;
+        pump_priv = priv->pump_private;
+
+        LOCK (&pump_priv->pump_state_lock);
+        {
+                ret = pump_priv->pump_state;
+        }
+        UNLOCK (&pump_priv->pump_state_lock);
+
+        return ret;
+}
+
+int
+pump_change_state (xlator_t *this, pump_state_t state)
+{
+        afr_private_t *priv = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        pump_state_t state_old;
+        pump_state_t state_new;
+
+
+        priv = this->private;
+        pump_priv = priv->pump_private;
+
+        GF_ASSERT (pump_priv);
+
+        LOCK (&pump_priv->pump_state_lock);
+        {
+                state_old = pump_priv->pump_state;
+                state_new = state;
+
+                pump_priv->pump_state = state;
+
+        }
+        UNLOCK (&pump_priv->pump_state_lock);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Pump changing state from %d to %d",
+                state_old,
+                state_new);
+
+        return  0;
+}
+
+static int
+pump_set_resume_path (xlator_t *this, const char *path)
+{
+        int ret = 0;
+
+        afr_private_t *priv = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        priv = this->private;
+        pump_priv = priv->pump_private;
+
+        GF_ASSERT (pump_priv);
+
+        LOCK (&pump_priv->resume_path_lock);
+        {
+                strncpy (pump_priv->resume_path, path, strlen (path) + 1);
+        }
+        UNLOCK (&pump_priv->resume_path_lock);
+
+        return ret;
+}
+
+static int
+pump_save_path (xlator_t *this, const char *path)
+{
+        afr_private_t *priv = NULL;
+        pump_state_t state;
+        dict_t *dict = NULL;
+        loc_t  loc = {0};
+        int dict_ret = 0;
+        int ret = -1;
+
+        state = pump_get_state ();
+        if (state == PUMP_STATE_RESUME)
+                return 0;
+
+        priv = this->private;
+
+        GF_ASSERT (priv->root_inode);
+
+        afr_build_root_loc (this, &loc);
+
+        dict = dict_new ();
+        dict_ret = dict_set_str (dict, PUMP_PATH, (char *)path);
+        if (dict_ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set the key %s", path, PUMP_PATH);
+
+        ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0);
+
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "setxattr failed - could not save path=%s", path);
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "setxattr succeeded - saved path=%s", path);
+        }
+
+        dict_unref (dict);
+
+        loc_wipe (&loc);
+        return 0;
+}
+
+static int
+pump_check_and_update_status (xlator_t *this)
+{
+        pump_state_t state;
+        int ret = -1;
+
+        state = pump_get_state ();
+
+        switch (state) {
+
+        case PUMP_STATE_RESUME:
+        case PUMP_STATE_RUNNING:
+        {
+                ret = 0;
+                break;
+        }
+        case PUMP_STATE_PAUSE:
+        {
+                ret = -1;
+                break;
+        }
+        case PUMP_STATE_ABORT:
+        {
+                pump_save_path (this, "/");
+                ret = -1;
+                break;
+        }
+        default:
+        {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unknown pump state");
+                ret = -1;
+                break;
+        }
+
+        }
+
+        return ret;
+}
+
+static const char *
+pump_get_resume_path (xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        const char *resume_path = NULL;
+
+        priv = this->private;
+        pump_priv = priv->pump_private;
+
+        resume_path = pump_priv->resume_path;
+
+        return resume_path;
+}
+
+static int
+pump_update_resume_state (xlator_t *this, const char *path)
+{
+        pump_state_t state;
+        const char *resume_path = NULL;
+
+        state = pump_get_state ();
+
+        if (state == PUMP_STATE_RESUME) {
+                resume_path = pump_get_resume_path (this);
+                if (strcmp (resume_path, "/") == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Reached the resume path (/). Proceeding to change state"
+                                " to running");
+                        pump_change_state (this, PUMP_STATE_RUNNING);
+                } else if (strcmp (resume_path, path) == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Reached the resume path. Proceeding to change state"
+                                " to running");
+                        pump_change_state (this, PUMP_STATE_RUNNING);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Not yet hit the resume path:res-path=%s,path=%s",
+                                resume_path, path);
+                }
+        }
+
+        return 0;
+}
+
+static gf_boolean_t
+is_pump_traversal_allowed (xlator_t *this, const char *path)
+{
+        pump_state_t state;
+        const char *resume_path = NULL;
+        gf_boolean_t ret = _gf_true;
+
+        state = pump_get_state ();
+
+        if (state == PUMP_STATE_RESUME) {
+                resume_path = pump_get_resume_path (this);
+                if (strstr (resume_path, path)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "On the right path to resumption path");
+                        ret = _gf_true;
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Not the right path to resuming=> ignoring traverse");
+                        ret = _gf_false;
+                }
+        }
+
+        return ret;
+}
+
+static int
+pump_save_file_stats (xlator_t *this, const char *path)
+{
+        afr_private_t  *priv        = NULL;
+        pump_private_t *pump_priv   = NULL;
+
+        priv      = this->private;
+        pump_priv = priv->pump_private;
+
+        LOCK (&pump_priv->resume_path_lock);
+        {
+                pump_priv->number_files_pumped++;
+
+                strncpy (pump_priv->current_file, path,
+                         PATH_MAX);
+        }
+        UNLOCK (&pump_priv->resume_path_lock);
+
+        return 0;
+}
+
+static int
+gf_pump_traverse_directory (loc_t *loc)
+{
+        xlator_t        *this              = NULL;
+        fd_t            *fd                = NULL;
+        off_t           offset             = 0;
+        loc_t           entry_loc          = {0};
+        gf_dirent_t     *entry             = NULL;
+        gf_dirent_t     *tmp               = NULL;
+        gf_dirent_t     entries;
+	struct iatt     iatt               = {0};
+        struct iatt     parent             = {0};
+	dict_t          *xattr_rsp         = NULL;
+        int             ret                = 0;
+        gf_boolean_t    is_directory_empty = _gf_true;
+        gf_boolean_t    free_entries       = _gf_false;
+
+        INIT_LIST_HEAD (&entries.list);
+        this = THIS;
+
+        GF_ASSERT (loc->inode);
+
+	fd = fd_create (loc->inode, pump_pid);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create fd for %s", loc->path);
+                goto out;
+        }
+
+        ret = syncop_opendir (this, loc, fd);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "opendir failed on %s", loc->path);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "pump opendir on %s returned=%d",
+                loc->path, ret);
+
+        while (syncop_readdirp (this, fd, 131072, offset, NULL, &entries)) {
+                free_entries = _gf_true;
+
+                if (list_empty (&entries.list)) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "no more entries in directory");
+                        goto out;
+                }
+
+                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "found readdir entry=%s", entry->d_name);
+
+                        offset = entry->d_off;
+                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
+                                gf_log (this->name, GF_LOG_WARNING, "%s/%s: No "
+                                        "gfid present skipping",
+                                        loc->path, entry->d_name);
+                                continue;
+                        }
+                        loc_wipe (&entry_loc);
+                        ret = afr_build_child_loc (this, &entry_loc, loc,
+                                                   entry->d_name);
+                        if (ret)
+                                goto out;
+
+                        if (!IS_ENTRY_CWD (entry->d_name) &&
+                            !IS_ENTRY_PARENT (entry->d_name)) {
+
+                                    is_directory_empty = _gf_false;
+                                    gf_log (this->name, GF_LOG_DEBUG,
+                                            "lookup %s => %"PRId64,
+                                            entry_loc.path,
+                                            iatt.ia_ino);
+
+                                    ret = syncop_lookup (this, &entry_loc, NULL,
+                                                         &iatt, &xattr_rsp, &parent);
+
+                                    if (ret) {
+                                            gf_log (this->name, GF_LOG_ERROR,
+                                                    "%s: lookup failed",
+                                                    entry_loc.path);
+                                            continue;
+                                    }
+                                    pump_fill_loc_info (&entry_loc, &iatt,
+                                                       &parent);
+
+                                    pump_update_resume_state (this, entry_loc.path);
+
+                                    pump_save_path (this, entry_loc.path);
+                                    pump_save_file_stats (this, entry_loc.path);
+
+                                    ret = pump_check_and_update_status (this);
+                                    if (ret < 0) {
+                                            gf_log (this->name, GF_LOG_DEBUG,
+                                                    "Pump beginning to exit out");
+                                            goto out;
+                                    }
+
+                                    if (IA_ISDIR (iatt.ia_type)) {
+                                            if (is_pump_traversal_allowed (this, entry_loc.path)) {
+                                                    gf_log (this->name, GF_LOG_TRACE,
+                                                            "entering dir=%s",
+                                                            entry->d_name);
+                                                    gf_pump_traverse_directory (&entry_loc);
+                                            }
+                                    }
+                        }
+                }
+
+                gf_dirent_free (&entries);
+                free_entries = _gf_false;
+                gf_log (this->name, GF_LOG_TRACE,
+                        "offset incremented to %d",
+                        (int32_t ) offset);
+
+        }
+
+        ret = syncop_close (fd);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_DEBUG, "closing the fd failed");
+
+        if (is_directory_empty && IS_ROOT_PATH (loc->path)) {
+               pump_change_state (this, PUMP_STATE_RUNNING);
+               gf_log (this->name, GF_LOG_INFO, "Empty source brick. "
+                                "Nothing to be done.");
+        }
+
+out:
+        if (entry_loc.path)
+                loc_wipe (&entry_loc);
+        if (free_entries)
+                gf_dirent_free (&entries);
+        return 0;
+}
+
+static int
+pump_update_resume_path (xlator_t *this)
+{
+        const char *resume_path = NULL;
+
+        resume_path = pump_get_resume_path (this);
+
+        if (resume_path) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Found a path to resume from: %s",
+                        resume_path);
+
+        }else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Did not find a path=> setting to '/'");
+                pump_set_resume_path (this, "/");
+        }
+
+        pump_change_state (this, PUMP_STATE_RESUME);
+
+        return 0;
+}
+
+static int32_t
+pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_private_t  *priv      = NULL;
+        loc_t           loc       = {0};
+        int             i         = 0;
+        int             ret       = 0;
+        int             source    = 0;
+        int             sink      = 1;
+
+        priv      = this->private;
+
+        afr_build_root_loc (this, &loc);
+
+        ret = syncop_removexattr (priv->children[source], &loc,
+                                          PUMP_PATH);
+
+        ret = syncop_removexattr (priv->children[sink], &loc,
+                                  PUMP_SINK_COMPLETE);
+
+        for (i = 0; i < priv->child_count; i++) {
+                ret = syncop_removexattr (priv->children[i], &loc,
+                                          PUMP_SOURCE_COMPLETE);
+                if (ret)
+                        gf_log (this->name, GF_LOG_DEBUG, "removexattr "
+                                "failed with %s", strerror (errno));
+        }
+
+        loc_wipe (&loc);
+        return pump_command_reply (frame, this);
+}
+
+static int
+pump_complete_migration (xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        pump_private_t *pump_priv = NULL;
+        dict_t *dict = NULL;
+        pump_state_t state;
+        loc_t  loc = {0};
+        int dict_ret = 0;
+        int ret = -1;
+
+        priv = this->private;
+        pump_priv = priv->pump_private;
+
+        GF_ASSERT (priv->root_inode);
+
+        afr_build_root_loc (this, &loc);
+
+        dict = dict_new ();
+
+        state = pump_get_state ();
+        if (state == PUMP_STATE_RUNNING) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Pump finished pumping");
+
+                pump_priv->pump_finished = _gf_true;
+
+                dict_ret = dict_set_str (dict, PUMP_SOURCE_COMPLETE, "jargon");
+                if (dict_ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set the key %s",
+                                loc.path, PUMP_SOURCE_COMPLETE);
+
+                ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "setxattr failed - while  notifying source complete");
+                }
+                dict_ret = dict_set_str (dict, PUMP_SINK_COMPLETE, "jargon");
+                if (dict_ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set the key %s",
+                                loc.path, PUMP_SINK_COMPLETE);
+
+                ret = syncop_setxattr (PUMP_SINK_CHILD (this), &loc, dict, 0);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "setxattr failed - while notifying sink complete");
+                }
+
+                pump_save_path (this, "/");
+
+        } else if (state == PUMP_STATE_ABORT) {
+                gf_log (this->name, GF_LOG_DEBUG, "Starting cleanup "
+                        "of pump internal xattrs");
+                call_resume (pump_priv->cleaner);
+        }
+
+        loc_wipe (&loc);
+        return 0;
+}
+
+static int
+pump_lookup_sink (loc_t *loc)
+{
+        xlator_t *this = NULL;
+	struct iatt iatt, parent;
+	dict_t *xattr_rsp;
+        dict_t *xattr_req = NULL;
+        int ret = 0;
+
+        this = THIS;
+
+        xattr_req = dict_new ();
+
+        ret = afr_set_root_gfid (xattr_req);
+        if (ret)
+                goto out;
+
+        ret = syncop_lookup (PUMP_SINK_CHILD (this), loc,
+                             xattr_req, &iatt, &xattr_rsp, &parent);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Lookup on sink child failed");
+                goto out;
+        }
+
+out:
+        if (xattr_req)
+                dict_unref (xattr_req);
+
+        return ret;
+}
+
+static int
+pump_task (void *data)
+{
+	xlator_t *this = NULL;
+        afr_private_t *priv = NULL;
+
+
+        loc_t loc = {0};
+	struct iatt iatt, parent;
+	dict_t *xattr_rsp = NULL;
+        dict_t *xattr_req = NULL;
+
+        int ret = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (priv->root_inode);
+
+        afr_build_root_loc (this, &loc);
+        xattr_req = dict_new ();
+        if (!xattr_req) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Out of memory");
+                ret = -1;
+                goto out;
+        }
+
+        afr_set_root_gfid (xattr_req);
+        ret = syncop_lookup (this, &loc, xattr_req,
+                             &iatt, &xattr_rsp, &parent);
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "lookup: path=%s gfid=%s",
+                loc.path, uuid_utoa (loc.inode->gfid));
+
+        ret = pump_check_and_update_status (this);
+        if (ret < 0) {
+                goto out;
+        }
+
+        pump_update_resume_path (this);
+
+        afr_set_root_gfid (xattr_req);
+        ret = pump_lookup_sink (&loc);
+        if (ret) {
+                pump_update_resume_path (this);
+                goto out;
+        }
+
+        gf_pump_traverse_directory (&loc);
+
+        pump_complete_migration (this);
+out:
+        if (xattr_req)
+                dict_unref (xattr_req);
+
+        loc_wipe (&loc);
+	return 0;
+}
+
+
+static int
+pump_task_completion (int ret, call_frame_t *sync_frame, void *data)
+{
+        xlator_t *this = NULL;
+        afr_private_t *priv = NULL;
+
+        this = THIS;
+
+        priv = this->private;
+
+        inode_unref (priv->root_inode);
+        STACK_DESTROY (sync_frame->root);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Pump xlator exiting");
+	return 0;
+}
+
+int
+pump_start (call_frame_t *pump_frame, xlator_t *this)
+{
+	afr_private_t *priv = NULL;
+	pump_private_t *pump_priv = NULL;
+
+	int ret = -1;
+
+	priv = this->private;
+        pump_priv = priv->pump_private;
+
+        afr_set_lk_owner (pump_frame, this, pump_frame->root);
+	pump_pid = (uint64_t) (unsigned long)pump_frame->root;
+
+	ret = synctask_new (pump_priv->env, pump_task,
+                            pump_task_completion,
+                            pump_frame, NULL);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "starting pump failed");
+                pump_change_state (this, PUMP_STATE_ABORT);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "setting pump as started lk_owner: %s %"PRIu64,
+                lkowner_utoa (&pump_frame->root->lk_owner), pump_pid);
+
+        priv->use_afr_in_pump = 1;
+out:
+	return ret;
+}
+
+static int
+pump_start_synctask (xlator_t *this)
+{
+        call_frame_t *frame = NULL;
+        int ret = 0;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Out of memory");
+                ret = -1;
+                goto out;
+        }
+
+        pump_change_state (this, PUMP_STATE_RUNNING);
+
+        ret = pump_start (frame, this);
+
+out:
+        return ret;
+}
+
+int32_t
+pump_cmd_start_setxattr_cbk (call_frame_t *frame,
+                             void *cookie,
+                             xlator_t *this,
+                             int32_t op_ret,
+                             int32_t op_errno, dict_t *xdata)
+
+{
+        call_frame_t *prev = NULL;
+        afr_local_t *local = NULL;
+        int ret = 0;
+
+        local = frame->local;
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not initiate destination "
+                        "brick connect");
+                ret = op_ret;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Successfully initiated destination "
+                "brick connect");
+
+        pump_mark_start_pending (this);
+
+        /* send the PARENT_UP as pump is ready now */
+        prev = cookie;
+        if (prev && prev->this)
+                prev->this->notify (prev->this, GF_EVENT_PARENT_UP, this);
+
+out:
+        local->op_ret = ret;
+        pump_command_reply (frame, this);
+
+        return 0;
+}
+
+static int
+pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local     = NULL;
+        afr_private_t *priv      = NULL;
+        dict_t        *dict      = NULL;
+        data_t        *data      = NULL;
+        char          *clnt_cmd  = NULL;
+        loc_t loc = {0};
+
+        int ret = 0;
+
+        priv  = this->private;
+        local = frame->local;
+
+        GF_ASSERT (priv->root_inode);
+
+        afr_build_root_loc (this, &loc);
+
+        data = data_ref (dict_get (local->dict, RB_PUMP_CMD_START));
+        if (!data) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not get destination brick value");
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        clnt_cmd = GF_CALLOC (1, data->len+1, gf_common_mt_char);
+        if (!clnt_cmd) {
+                ret = -1;
+                goto out;
+        }
+
+        memcpy (clnt_cmd, data->data, data->len);
+        clnt_cmd[data->len] = '\0';
+        gf_log (this->name, GF_LOG_DEBUG, "Got destination brick %s\n",
+                        clnt_cmd);
+
+        ret = dict_set_dynstr (dict, CLIENT_CMD_CONNECT, clnt_cmd);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not inititiate destination brick "
+                        "connect");
+                goto out;
+        }
+
+	STACK_WIND (frame,
+		    pump_cmd_start_setxattr_cbk,
+		    PUMP_SINK_CHILD(this),
+		    PUMP_SINK_CHILD(this)->fops->setxattr,
+		    &loc,
+		    dict,
+		    0, NULL);
+
+        ret = 0;
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        if (data)
+                data_unref (data);
+
+        if (ret && clnt_cmd)
+                GF_FREE (clnt_cmd);
+
+        loc_wipe (&loc);
+        return ret;
+}
+
+static int
+is_pump_aborted (xlator_t *this)
+{
+        pump_state_t state;
+
+        state = pump_get_state ();
+
+        return ((state == PUMP_STATE_ABORT));
+}
+
+int32_t
+pump_cmd_start_getxattr_cbk (call_frame_t *frame,
+                             void *cookie,
+                             xlator_t *this,
+                             int32_t op_ret,
+                             int32_t op_errno,
+                             dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        char *path = NULL;
+
+        pump_state_t state;
+        int ret = 0;
+        int need_unwind = 0;
+        int dict_ret = -1;
+
+        local = frame->local;
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "getxattr failed - changing pump "
+                        "state to RUNNING with '/'");
+                path = "/";
+                ret = op_ret;
+        } else {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "getxattr succeeded");
+
+                dict_ret =  dict_get_str (dict, PUMP_PATH, &path);
+                if (dict_ret < 0)
+                        path = "/";
+        }
+
+        state = pump_get_state ();
+        if ((state == PUMP_STATE_RUNNING) ||
+            (state == PUMP_STATE_RESUME)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pump is already started");
+                ret = -1;
+                goto out;
+        }
+
+        pump_set_resume_path (this, path);
+
+        if (is_pump_aborted (this))
+                /* We're re-starting pump afresh */
+                ret = pump_initiate_sink_connect (frame, this);
+        else {
+                /* We're re-starting pump from a previous
+                   pause */
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "about to start synctask");
+                ret = pump_start_synctask (this);
+                need_unwind = 1;
+        }
+
+out:
+        if ((ret < 0) || (need_unwind == 1)) {
+                local->op_ret = ret;
+                pump_command_reply (frame, this);
+        }
+	return 0;
+}
+
+int
+pump_execute_status (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        uint64_t number_files = 0;
+
+        char filename[PATH_MAX];
+        char summary[PATH_MAX+256];
+        char *dict_str = NULL;
+
+        int32_t op_ret = 0;
+        int32_t op_errno = 0;
+
+        dict_t *dict = NULL;
+        int ret = -1;
+
+        priv = this->private;
+        pump_priv = priv->pump_private;
+
+        LOCK (&pump_priv->resume_path_lock);
+        {
+                number_files  = pump_priv->number_files_pumped;
+                strncpy (filename, pump_priv->current_file, PATH_MAX);
+        }
+        UNLOCK (&pump_priv->resume_path_lock);
+
+        dict_str     = GF_CALLOC (1, PATH_MAX + 256, gf_afr_mt_char);
+        if (!dict_str) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Out of memory");
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        if (pump_priv->pump_finished) {
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64, number_files);
+        } else {
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64":current_file=%s",
+                          number_files, filename);
+        }
+        snprintf (dict_str, PATH_MAX+256, "status=%d:%s",
+                  (pump_priv->pump_finished)?1:0, summary);
+
+        dict = dict_new ();
+
+        ret = dict_set_dynstr (dict, RB_PUMP_CMD_STATUS, dict_str);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dict_set_dynstr returned negative value");
+        } else {
+                dict_str = NULL;
+        }
+
+        op_ret = 0;
+
+out:
+
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
+
+        if (dict)
+                dict_unref (dict);
+
+        GF_FREE (dict_str);
+
+        return 0;
+}
+
+int
+pump_execute_pause (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        pump_change_state (this, PUMP_STATE_PAUSE);
+
+        local->op_ret = 0;
+        pump_command_reply (frame, this);
+
+        return 0;
+}
+
+int
+pump_execute_start (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t   *local = NULL;
+
+        int ret = 0;
+        loc_t loc = {0};
+
+        priv = this->private;
+        local = frame->local;
+
+        if (!priv->root_inode) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pump xlator cannot be started without an initial "
+                        "lookup");
+                ret = -1;
+                goto out;
+        }
+
+        GF_ASSERT (priv->root_inode);
+
+        afr_build_root_loc (this, &loc);
+
+	STACK_WIND (frame,
+		    pump_cmd_start_getxattr_cbk,
+		    PUMP_SOURCE_CHILD(this),
+		    PUMP_SOURCE_CHILD(this)->fops->getxattr,
+		    &loc,
+		    PUMP_PATH, NULL);
+
+        ret = 0;
+
+out:
+        if (ret < 0) {
+                local->op_ret = ret;
+                pump_command_reply (frame, this);
+        }
+
+        loc_wipe (&loc);
+	return 0;
+}
+
+static int
+pump_cleanup_helper (void *data) {
+        call_frame_t *frame = data;
+
+        pump_xattr_cleaner (frame, 0, frame->this, 0, 0, NULL);
+
+        return 0;
+}
+
+static int
+pump_cleanup_done (int ret, call_frame_t *sync_frame, void *data)
+{
+        STACK_DESTROY (sync_frame->root);
+
+        return 0;
+}
+
+int
+pump_execute_commit (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t  *priv       = NULL;
+        pump_private_t *pump_priv  = NULL;
+        afr_local_t    *local      = NULL;
+        call_frame_t   *sync_frame = NULL;
+        int             ret        = 0;
+
+        priv      = this->private;
+        pump_priv = priv->pump_private;
+        local     = frame->local;
+
+        local->op_ret = 0;
+        if (pump_priv->pump_finished) {
+                pump_change_state (this, PUMP_STATE_COMMIT);
+                sync_frame = create_frame (this, this->ctx->pool);
+                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
+                                    pump_cleanup_done, sync_frame, frame);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Couldn't create "
+                                "synctask for cleaning up xattrs.");
+                }
+
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Commit can't proceed. "
+                        "Migration in progress");
+                local->op_ret = -1;
+                local->op_errno = EINPROGRESS;
+                pump_command_reply (frame, this);
+        }
+
+        return 0;
+}
+int
+pump_execute_abort (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t  *priv       = NULL;
+        pump_private_t *pump_priv  = NULL;
+        afr_local_t    *local      = NULL;
+        call_frame_t   *sync_frame = NULL;
+        int             ret        = 0;
+
+        priv      = this->private;
+        pump_priv = priv->pump_private;
+        local     = frame->local;
+
+        pump_change_state (this, PUMP_STATE_ABORT);
+
+        LOCK (&pump_priv->resume_path_lock);
+        {
+                pump_priv->number_files_pumped = 0;
+                pump_priv->current_file[0] = '\0';
+        }
+        UNLOCK (&pump_priv->resume_path_lock);
+
+        local->op_ret = 0;
+        if (pump_priv->pump_finished) {
+                sync_frame = create_frame (this, this->ctx->pool);
+                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
+                                    pump_cleanup_done, sync_frame, frame);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Couldn't create "
+                                "synctask for cleaning up xattrs.");
+                }
+
+        } else {
+                pump_priv->cleaner = fop_setxattr_cbk_stub (frame,
+                                                            pump_xattr_cleaner,
+                                                            0, 0, NULL);
+        }
+
+        return 0;
+}
+
+gf_boolean_t
+pump_command_status (xlator_t *this, dict_t *dict)
+{
+        char *cmd = NULL;
+        int dict_ret = -1;
+        int ret = _gf_true;
+
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_STATUS, &cmd);
+        if (dict_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not a pump status command");
+                ret = _gf_false;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Hit a pump command - status");
+        ret = _gf_true;
+
+out:
+        return ret;
+
+}
+
+gf_boolean_t
+pump_command_pause (xlator_t *this, dict_t *dict)
+{
+        char *cmd = NULL;
+        int dict_ret = -1;
+        int ret = _gf_true;
+
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_PAUSE, &cmd);
+        if (dict_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not a pump pause command");
+                ret = _gf_false;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Hit a pump command - pause");
+        ret = _gf_true;
+
+out:
+        return ret;
+
+}
+
+gf_boolean_t
+pump_command_commit (xlator_t *this, dict_t *dict)
+{
+        char *cmd = NULL;
+        int dict_ret = -1;
+        int ret = _gf_true;
+
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_COMMIT, &cmd);
+        if (dict_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not a pump commit command");
+                ret = _gf_false;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Hit a pump command - commit");
+        ret = _gf_true;
+
+out:
+        return ret;
+
+}
+
+gf_boolean_t
+pump_command_abort (xlator_t *this, dict_t *dict)
+{
+        char *cmd = NULL;
+        int dict_ret = -1;
+        int ret = _gf_true;
+
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_ABORT, &cmd);
+        if (dict_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not a pump abort command");
+                ret = _gf_false;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Hit a pump command - abort");
+        ret = _gf_true;
+
+out:
+        return ret;
+
+}
+
+gf_boolean_t
+pump_command_start (xlator_t *this, dict_t *dict)
+{
+        char *cmd = NULL;
+        int dict_ret = -1;
+        int ret = _gf_true;
+
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_START, &cmd);
+        if (dict_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not a pump start command");
+                ret = _gf_false;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Hit a pump command - start");
+        ret = _gf_true;
+
+out:
+        return ret;
+
+}
+
+struct _xattr_key {
+        char *key;
+        struct list_head list;
+};
+
+static int
+__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
+                     void *data)
+{
+        struct list_head *  list  = data;
+        struct _xattr_key * xkey  = NULL;
+
+        if (!strncmp (key, AFR_XATTR_PREFIX,
+                      strlen (AFR_XATTR_PREFIX))) {
+
+                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
+                if (!xkey)
+                        return -1;
+
+                xkey->key = key;
+                INIT_LIST_HEAD (&xkey->list);
+
+                list_add_tail (&xkey->list, list);
+        }
+        return 0;
+}
+
+static void
+__filter_xattrs (dict_t *dict)
+{
+        struct list_head keys;
+
+        struct _xattr_key *key;
+        struct _xattr_key *tmp;
+
+        INIT_LIST_HEAD (&keys);
+
+        dict_foreach (dict, __gather_xattr_keys,
+                      (void *) &keys);
+
+        list_for_each_entry_safe (key, tmp, &keys, list) {
+                dict_del (dict, key->key);
+
+                list_del_init (&key->list);
+
+                GF_FREE (key);
+        }
+}
+
+int32_t
+pump_getxattr_cbk (call_frame_t *frame, void *cookie,
+		  xlator_t *this, int32_t op_ret, int32_t op_errno,
+		  dict_t *dict, dict_t *xdata)
+{
+	afr_private_t   *priv           = NULL;
+	afr_local_t     *local          = NULL;
+	xlator_t        **children      = NULL;
+	int             unwind          = 1;
+        int32_t         *last_index     = NULL;
+        int32_t         next_call_child = -1;
+        int32_t         read_child      = -1;
+        int32_t         *fresh_children = NULL;
+
+
+	priv     = this->private;
+	children = priv->children;
+
+	local = frame->local;
+
+        read_child = (long) cookie;
+
+	if (op_ret == -1) {
+		last_index = &local->cont.getxattr.last_index;
+                fresh_children = local->fresh_children;
+                next_call_child = afr_next_call_child (fresh_children,
+                                                       local->child_up,
+                                                       priv->child_count,
+                                                       last_index, read_child);
+                if (next_call_child < 0)
+                        goto out;
+
+		unwind = 0;
+		STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
+				   (void *) (long) read_child,
+				   children[next_call_child],
+				   children[next_call_child]->fops->getxattr,
+				   &local->loc,
+				   local->cont.getxattr.name, NULL);
+	}
+
+out:
+	if (unwind) {
+                if (op_ret >= 0 && dict)
+                        __filter_xattrs (dict);
+
+		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
+	}
+
+	return 0;
+}
+
+int32_t
+pump_getxattr (call_frame_t *frame, xlator_t *this,
+	      loc_t *loc, const char *name, dict_t *xdata)
+{
+	afr_private_t *   priv       = NULL;
+	xlator_t **       children   = NULL;
+	int               call_child = 0;
+	afr_local_t       *local     = NULL;
+	int32_t           ret     = -1;
+	int32_t           op_errno   = 0;
+        uint64_t          read_child = 0;
+
+
+	VALIDATE_OR_GOTO (frame, out);
+	VALIDATE_OR_GOTO (this, out);
+	VALIDATE_OR_GOTO (this->private, out);
+
+	priv     = this->private;
+	VALIDATE_OR_GOTO (priv->children, out);
+
+	children = priv->children;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame, default_getxattr_cbk,
+                            FIRST_CHILD (this),
+                            (FIRST_CHILD (this))->fops->getxattr,
+                            loc, name, xdata);
+                return 0;
+        }
+
+
+	AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+	local = frame->local;
+
+        ret = afr_local_init (local, priv, &op_errno);
+        if (ret < 0)
+                goto out;
+
+        if (name) {
+                if (!strncmp (name, AFR_XATTR_PREFIX,
+                              strlen (AFR_XATTR_PREFIX))) {
+
+                        op_errno = ENODATA;
+                        goto out;
+                }
+
+                if (!strcmp (name, RB_PUMP_CMD_STATUS)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Hit pump command - status");
+                        pump_execute_status (frame, this);
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        local->fresh_children = GF_CALLOC (priv->child_count,
+                                          sizeof (*local->fresh_children),
+                                          gf_afr_mt_int32_t);
+        if (!local->fresh_children) {
+                ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        read_child = afr_inode_get_read_ctx (this, loc->inode, local->fresh_children);
+        ret = afr_get_call_child (this, local->child_up, read_child,
+                                     local->fresh_children,
+                                     &call_child,
+                                     &local->cont.getxattr.last_index);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
+	loc_copy (&local->loc, loc);
+	if (name)
+	  local->cont.getxattr.name       = gf_strdup (name);
+
+	STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
+			   (void *) (long) call_child,
+			   children[call_child], children[call_child]->fops->getxattr,
+			   loc, name, xdata);
+
+	ret = 0;
+out:
+	if (ret < 0)
+		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+	return 0;
+}
+
+static int
+afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *   local = NULL;
+	call_frame_t   *main_frame = NULL;
+
+	local = frame->local;
+
+	LOCK (&frame->lock);
+	{
+		if (local->transaction.main_frame)
+			main_frame = local->transaction.main_frame;
+		local->transaction.main_frame = NULL;
+	}
+	UNLOCK (&frame->lock);
+
+	if (main_frame) {
+		AFR_STACK_UNWIND (setxattr, main_frame,
+                                  local->op_ret, local->op_errno, NULL);
+	}
+	return 0;
+}
+
+static int
+afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	afr_local_t *   local = NULL;
+	afr_private_t * priv  = NULL;
+
+	int call_count  = -1;
+	int need_unwind = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	LOCK (&frame->lock);
+	{
+		if (op_ret != -1) {
+			if (local->success_count == 0) {
+				local->op_ret = op_ret;
+			}
+			local->success_count++;
+
+			if (local->success_count == priv->child_count) {
+				need_unwind = 1;
+			}
+		}
+
+		local->op_errno = op_errno;
+	}
+	UNLOCK (&frame->lock);
+
+	if (need_unwind)
+		local->transaction.unwind (frame, this);
+
+	call_count = afr_frame_return (frame);
+
+	if (call_count == 0) {
+		local->transaction.resume (frame, this);
+	}
+
+	return 0;
+}
+
+static int
+afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	int call_count = -1;
+	int i = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	call_count = afr_up_children_count (local->child_up, priv->child_count);
+
+	if (call_count == 0) {
+		local->transaction.resume (frame, this);
+		return 0;
+	}
+
+	local->call_count = call_count;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->child_up[i]) {
+			STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
+					   (void *) (long) i,
+					   priv->children[i],
+					   priv->children[i]->fops->setxattr,
+					   &local->loc,
+					   local->cont.setxattr.dict,
+					   local->cont.setxattr.flags, NULL);
+
+			if (!--call_count)
+				break;
+		}
+	}
+
+	return 0;
+}
+
+
+static int
+afr_setxattr_done (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t * local = frame->local;
+
+	local->transaction.unwind (frame, this);
+
+	AFR_STACK_DESTROY (frame);
+
+	return 0;
+}
+
+int32_t
+pump_setxattr_cbk (call_frame_t *frame,
+		      void *cookie,
+		      xlator_t *this,
+		      int32_t op_ret,
+		      int32_t op_errno, dict_t *xdata)
+{
+	AFR_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+	return 0;
+}
+
+int
+pump_command_reply (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (local->op_ret < 0)
+                gf_log (this->name, GF_LOG_INFO,
+                        "Command failed");
+        else
+                gf_log (this->name, GF_LOG_INFO,
+                        "Command succeeded");
+
+        AFR_STACK_UNWIND (setxattr,
+                          frame,
+                          local->op_ret,
+                          local->op_errno, NULL);
+
+        return 0;
+}
+
+int
+pump_parse_command (call_frame_t *frame, xlator_t *this,
+                    afr_local_t *local, dict_t *dict)
+{
+
+        int ret = -1;
+
+        if (pump_command_start (this, dict)) {
+                frame->local = local;
+                local->dict = dict_ref (dict);
+                ret = pump_execute_start (frame, this);
+
+        } else if (pump_command_pause (this, dict)) {
+                frame->local = local;
+                local->dict = dict_ref (dict);
+                ret = pump_execute_pause (frame, this);
+
+        } else if (pump_command_abort (this, dict)) {
+                frame->local = local;
+                local->dict = dict_ref (dict);
+                ret = pump_execute_abort (frame, this);
+
+        } else if (pump_command_commit (this, dict)) {
+                frame->local = local;
+                local->dict = dict_ref (dict);
+                ret = pump_execute_commit (frame, this);
+        }
+        return ret;
+}
+
+int
+pump_setxattr (call_frame_t *frame, xlator_t *this,
+               loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+{
+	afr_private_t * priv  = NULL;
+	afr_local_t   * local = NULL;
+	call_frame_t   *transaction_frame = NULL;
+	int ret = -1;
+	int op_errno = 0;
+
+	VALIDATE_OR_GOTO (frame, out);
+	VALIDATE_OR_GOTO (this, out);
+	VALIDATE_OR_GOTO (this->private, out);
+
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict,
+                                   op_errno, out);
+
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame, default_setxattr_cbk,
+                            FIRST_CHILD (this),
+                            (FIRST_CHILD (this))->fops->setxattr,
+                            loc, dict, flags, xdata);
+                return 0;
+        }
+
+
+	AFR_LOCAL_ALLOC_OR_GOTO (local, out);
+
+	ret = afr_local_init (local, priv, &op_errno);
+	if (ret < 0) {
+                afr_local_cleanup (local, this);
+		goto out;
+        }
+
+        ret = pump_parse_command (frame, this,
+                                  local, dict);
+        if (ret >= 0) {
+                ret = 0;
+                goto out;
+        }
+
+	transaction_frame = copy_frame (frame);
+	if (!transaction_frame) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Out of memory.");
+                op_errno = ENOMEM;
+                ret = -1;
+                afr_local_cleanup (local, this);
+		goto out;
+	}
+
+	transaction_frame->local = local;
+
+	local->op_ret = -1;
+
+	local->cont.setxattr.dict  = dict_ref (dict);
+	local->cont.setxattr.flags = flags;
+
+	local->transaction.fop    = afr_setxattr_wind;
+	local->transaction.done   = afr_setxattr_done;
+	local->transaction.unwind = afr_setxattr_unwind;
+
+	loc_copy (&local->loc, loc);
+
+	local->transaction.main_frame = frame;
+	local->transaction.start   = LLONG_MAX - 1;
+	local->transaction.len     = 0;
+
+	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+
+	ret = 0;
+out:
+	if (ret < 0) {
+		if (transaction_frame)
+			AFR_STACK_DESTROY (transaction_frame);
+		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+	}
+
+	return 0;
+}
+
+/* Defaults */
+static int32_t
+pump_lookup (call_frame_t *frame,
+             xlator_t *this,
+             loc_t *loc,
+             dict_t *xattr_req)
+{
+	afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_lookup_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->lookup,
+                            loc,
+                            xattr_req);
+                return 0;
+        }
+
+        afr_lookup (frame, this, loc, xattr_req);
+        return 0;
+}
+
+
+static int32_t
+pump_truncate (call_frame_t *frame,
+               xlator_t *this,
+               loc_t *loc,
+               off_t offset, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_truncate_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->truncate,
+                            loc,
+                            offset, xdata);
+                return 0;
+        }
+
+        afr_truncate (frame, this, loc, offset, xdata);
+        return 0;
+}
+
+
+static int32_t
+pump_ftruncate (call_frame_t *frame,
+                xlator_t *this,
+                fd_t *fd,
+                off_t offset, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_ftruncate_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->ftruncate,
+                            fd,
+                            offset, xdata);
+                return 0;
+        }
+
+        afr_ftruncate (frame, this, fd, offset, xdata);
+        return 0;
+}
+
+
+
+
+int
+pump_mknod (call_frame_t *frame, xlator_t *this,
+            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame, default_mknod_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->mknod,
+                            loc, mode, rdev, umask, xdata);
+                return 0;
+        }
+        afr_mknod (frame, this, loc, mode, rdev, umask, xdata);
+        return 0;
+
+}
+
+
+
+int
+pump_mkdir (call_frame_t *frame, xlator_t *this,
+            loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame, default_mkdir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->mkdir,
+                            loc, mode, umask, xdata);
+                return 0;
+        }
+        afr_mkdir (frame, this, loc, mode, umask, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_unlink (call_frame_t *frame,
+             xlator_t *this,
+             loc_t *loc, int xflag, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_unlink_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->unlink,
+                            loc, xflag, xdata);
+                return 0;
+        }
+        afr_unlink (frame, this, loc, xflag, xdata);
+        return 0;
+
+}
+
+
+static int
+pump_rmdir (call_frame_t *frame, xlator_t *this,
+            loc_t *loc, int flags, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+
+	priv = this->private;
+
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame, default_rmdir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->rmdir,
+                            loc, flags, xdata);
+                return 0;
+        }
+
+        afr_rmdir (frame, this, loc, flags, xdata);
+        return 0;
+
+}
+
+
+
+int
+pump_symlink (call_frame_t *frame, xlator_t *this,
+              const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame, default_symlink_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->symlink,
+                            linkpath, loc, umask, xdata);
+                return 0;
+        }
+        afr_symlink (frame, this, linkpath, loc, umask, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_rename (call_frame_t *frame,
+             xlator_t *this,
+             loc_t *oldloc,
+             loc_t *newloc, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_rename_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->rename,
+                            oldloc, newloc, xdata);
+                return 0;
+        }
+        afr_rename (frame, this, oldloc, newloc, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_link (call_frame_t *frame,
+           xlator_t *this,
+           loc_t *oldloc,
+           loc_t *newloc, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_link_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->link,
+                            oldloc, newloc, xdata);
+                return 0;
+        }
+        afr_link (frame, this, oldloc, newloc, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_create (call_frame_t *frame, xlator_t *this,
+             loc_t *loc, int32_t flags, mode_t mode,
+             mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame, default_create_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->create,
+                            loc, flags, mode, umask, fd, xdata);
+                return 0;
+        }
+        afr_create (frame, this, loc, flags, mode, umask, fd, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_open (call_frame_t *frame,
+           xlator_t *this,
+           loc_t *loc,
+           int32_t flags, fd_t *fd, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_open_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->open,
+                            loc, flags, fd, xdata);
+                return 0;
+        }
+        afr_open (frame, this, loc, flags, fd, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_writev (call_frame_t *frame,
+             xlator_t *this,
+             fd_t *fd,
+             struct iovec *vector,
+             int32_t count,
+             off_t off, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_writev_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->writev,
+                            fd,
+                            vector,
+                            count,
+                            off, flags,
+                            iobref, xdata);
+                return 0;
+        }
+
+        afr_writev (frame, this, fd, vector, count, off, flags, iobref, xdata);
+        return 0;
+}
+
+
+static int32_t
+pump_flush (call_frame_t *frame,
+            xlator_t *this,
+            fd_t *fd, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_flush_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->flush,
+                            fd, xdata);
+                return 0;
+        }
+        afr_flush (frame, this, fd, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_fsync (call_frame_t *frame,
+            xlator_t *this,
+            fd_t *fd,
+            int32_t flags, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_fsync_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsync,
+                            fd,
+                            flags, xdata);
+                return 0;
+        }
+        afr_fsync (frame, this, fd, flags, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_opendir (call_frame_t *frame,
+              xlator_t *this,
+              loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_opendir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->opendir,
+                            loc, fd, xdata);
+                return 0;
+        }
+        afr_opendir (frame, this, loc, fd, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_fsyncdir (call_frame_t *frame,
+               xlator_t *this,
+               fd_t *fd,
+               int32_t flags, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_fsyncdir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsyncdir,
+                            fd,
+                            flags, xdata);
+                return 0;
+        }
+        afr_fsyncdir (frame, this, fd, flags, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_xattrop (call_frame_t *frame,
+              xlator_t *this,
+              loc_t *loc,
+              gf_xattrop_flags_t flags,
+              dict_t *dict, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_xattrop_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->xattrop,
+                            loc,
+                            flags,
+                            dict, xdata);
+                return 0;
+        }
+        afr_xattrop (frame, this, loc, flags, dict, xdata);
+        return 0;
+
+}
+
+static int32_t
+pump_fxattrop (call_frame_t *frame,
+               xlator_t *this,
+               fd_t *fd,
+               gf_xattrop_flags_t flags,
+               dict_t *dict, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_fxattrop_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fxattrop,
+                            fd,
+                            flags,
+                            dict, xdata);
+                return 0;
+        }
+        afr_fxattrop (frame, this, fd, flags, dict, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_removexattr (call_frame_t *frame,
+                  xlator_t *this,
+                  loc_t *loc,
+                  const char *name, dict_t *xdata)
+{
+        afr_private_t *priv     = NULL;
+        int            op_errno = -1;
+
+        VALIDATE_OR_GOTO (this, out);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.pump*",
+                                 name, op_errno, out);
+
+        op_errno = 0;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_removexattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->removexattr,
+                            loc,
+                            name, xdata);
+                return 0;
+        }
+        afr_removexattr (frame, this, loc, name, xdata);
+
+ out:
+        if (op_errno)
+                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+        return 0;
+
+}
+
+
+
+static int32_t
+pump_readdir (call_frame_t *frame,
+              xlator_t *this,
+              fd_t *fd,
+              size_t size,
+              off_t off, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_readdir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->readdir,
+                            fd, size, off, xdata);
+                return 0;
+        }
+        afr_readdir (frame, this, fd, size, off, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               size_t size, off_t off, dict_t *dict)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_readdirp_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->readdirp,
+                            fd, size, off, dict);
+                return 0;
+        }
+        afr_readdirp (frame, this, fd, size, off, dict);
+        return 0;
+
+}
+
+
+
+static int32_t
+pump_releasedir (xlator_t *this,
+                 fd_t *fd)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (priv->use_afr_in_pump)
+                afr_releasedir (this, fd);
+	return 0;
+
+}
+
+static int32_t
+pump_release (xlator_t *this,
+              fd_t *fd)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (priv->use_afr_in_pump)
+                afr_release (this, fd);
+	return 0;
+
+}
+
+static int32_t
+pump_forget (xlator_t *this, inode_t *inode)
+{
+        afr_private_t  *priv  = NULL;
+
+        priv = this->private;
+        if (priv->use_afr_in_pump)
+                afr_forget (this, inode);
+
+        return 0;
+}
+
+static int32_t
+pump_setattr (call_frame_t *frame,
+              xlator_t *this,
+              loc_t *loc,
+              struct iatt *stbuf,
+              int32_t valid, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_setattr_cbk,
+                            FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->setattr,
+                            loc, stbuf, valid, xdata);
+                return 0;
+        }
+        afr_setattr (frame, this, loc, stbuf, valid, xdata);
+        return 0;
+
+}
+
+
+static int32_t
+pump_fsetattr (call_frame_t *frame,
+               xlator_t *this,
+               fd_t *fd,
+               struct iatt *stbuf,
+               int32_t valid, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+	priv = this->private;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame,
+                            default_fsetattr_cbk,
+                            FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->fsetattr,
+                            fd, stbuf, valid, xdata);
+                return 0;
+        }
+        afr_fsetattr (frame, this, fd, stbuf, valid, xdata);
+        return 0;
+
+}
+
+
+/* End of defaults */
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_afr_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+                                "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+static int
+is_xlator_pump_sink (xlator_t *child)
+{
+        return (child == PUMP_SINK_CHILD(THIS));
+}
+
+static int
+is_xlator_pump_source (xlator_t *child)
+{
+        return (child == PUMP_SOURCE_CHILD(THIS));
+}
+
+int32_t
+notify (xlator_t *this, int32_t event,
+	void *data, ...)
+{
+        int ret = -1;
+        xlator_t *child_xl = NULL;
+
+        child_xl = (xlator_t *) data;
+
+        ret = afr_notify (this, event, data, NULL);
+
+	switch (event) {
+	case GF_EVENT_CHILD_DOWN:
+                if (is_xlator_pump_source (child_xl))
+                        pump_change_state (this, PUMP_STATE_ABORT);
+                break;
+
+        case GF_EVENT_CHILD_UP:
+                if (is_xlator_pump_sink (child_xl))
+                        if (is_pump_start_pending (this)) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "about to start synctask");
+                                ret = pump_start_synctask (this);
+                                if (ret < 0)
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "Could not start pump "
+                                                "synctask");
+                                else
+                                        pump_remove_start_pending (this);
+                        }
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+	afr_private_t * priv        = NULL;
+        pump_private_t *pump_priv = NULL;
+	int             child_count = 0;
+	xlator_list_t * trav        = NULL;
+	int             i           = 0;
+	int             ret         = -1;
+	GF_UNUSED int   op_errno    = 0;
+
+        int source_child = 0;
+
+	if (!this->children) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"pump translator needs a source and sink"
+                        "subvolumes defined.");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Volume is dangling.");
+	}
+
+	this->private = GF_CALLOC (1, sizeof (afr_private_t),
+                                   gf_afr_mt_afr_private_t);
+        if (!this->private)
+                goto out;
+
+	priv = this->private;
+        LOCK_INIT (&priv->lock);
+        LOCK_INIT (&priv->read_child_lock);
+        //lock recovery is not done in afr
+        pthread_mutex_init (&priv->mutex, NULL);
+        INIT_LIST_HEAD (&priv->saved_fds);
+
+        child_count = xlator_subvolume_count (this);
+        if (child_count != 2) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "There should be exactly 2 children - one source "
+                        "and one sink");
+                return -1;
+        }
+	priv->child_count = child_count;
+
+        priv->read_child = source_child;
+        priv->favorite_child = source_child;
+        priv->background_self_heal_count = 0;
+
+	priv->data_self_heal     = "on";
+	priv->metadata_self_heal = 1;
+	priv->entry_self_heal    = 1;
+
+        priv->data_self_heal_window_size = 16;
+
+	priv->data_change_log     = 1;
+	priv->metadata_change_log = 1;
+	priv->entry_change_log    = 1;
+        priv->use_afr_in_pump = 1;
+        priv->sh_readdir_size = 65536;
+
+	/* Locking options */
+
+        /* Lock server count infact does not matter. Locks are held
+           on all subvolumes, in this case being the source
+           and the sink.
+        */
+
+	priv->strict_readdir = _gf_false;
+
+	priv->wait_count = 1;
+	priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
+                                 gf_afr_mt_char);
+	if (!priv->child_up) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Out of memory.");
+		op_errno = ENOMEM;
+		goto out;
+	}
+
+	priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
+                                 gf_afr_mt_xlator_t);
+	if (!priv->children) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Out of memory.");
+		op_errno = ENOMEM;
+		goto out;
+	}
+
+        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
+                                       child_count,
+                                       gf_afr_mt_char);
+        if (!priv->pending_key) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Out of memory.");
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+	trav = this->children;
+	i = 0;
+	while (i < child_count) {
+		priv->children[i] = trav->xlator;
+
+                ret = gf_asprintf (&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                                   trav->xlator->name);
+                if (-1 == ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "asprintf failed to set pending key");
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+
+		trav = trav->next;
+		i++;
+	}
+
+        ret = gf_asprintf (&priv->sh_domain, "%s-self-heal", this->name);
+        if (-1 == ret) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        priv->first_lookup = 1;
+        priv->root_inode = NULL;
+
+        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
+                                      gf_afr_mt_int32_t);
+        if (!priv->last_event) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+	pump_priv = GF_CALLOC (1, sizeof (*pump_priv),
+                            gf_afr_mt_pump_priv);
+	if (!pump_priv) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Out of memory");
+                op_errno = ENOMEM;
+		goto out;
+	}
+
+        LOCK_INIT (&pump_priv->resume_path_lock);
+        LOCK_INIT (&pump_priv->pump_state_lock);
+
+        pump_priv->resume_path = GF_CALLOC (1, PATH_MAX,
+                                            gf_afr_mt_char);
+        if (!pump_priv->resume_path) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Out of memory");
+                ret = -1;
+                goto out;
+        }
+
+	pump_priv->env = this->ctx->env;
+        if (!pump_priv->env) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not create new sync-environment");
+                ret = -1;
+                goto out;
+        }
+
+        /* keep more local here as we may need them for self-heal etc */
+        this->local_pool = mem_pool_new (afr_local_t, 128);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
+	priv->pump_private = pump_priv;
+
+        pump_change_state (this, PUMP_STATE_ABORT);
+
+	ret = 0;
+out:
+	return ret;
+}
+
+int
+fini (xlator_t *this)
+{
+        afr_private_t * priv        = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        priv      = this->private;
+        this->private = NULL;
+        if (!priv)
+                goto out;
+
+        pump_priv = priv->pump_private;
+        if (!pump_priv)
+                goto afr_priv;
+
+        GF_FREE (pump_priv->resume_path);
+        LOCK_DESTROY (&pump_priv->resume_path_lock);
+        LOCK_DESTROY (&pump_priv->pump_state_lock);
+        GF_FREE (pump_priv);
+afr_priv:
+        afr_priv_destroy (priv);
+out:
+	return 0;
+}
+
+
+struct xlator_fops fops = {
+	.lookup      = pump_lookup,
+	.open        = pump_open,
+	.flush       = pump_flush,
+	.fsync       = pump_fsync,
+	.fsyncdir    = pump_fsyncdir,
+	.xattrop     = pump_xattrop,
+	.fxattrop    = pump_fxattrop,
+        .getxattr    = pump_getxattr,
+
+	/* inode write */
+	.writev      = pump_writev,
+	.truncate    = pump_truncate,
+	.ftruncate   = pump_ftruncate,
+	.setxattr    = pump_setxattr,
+        .setattr     = pump_setattr,
+	.fsetattr    = pump_fsetattr,
+	.removexattr = pump_removexattr,
+
+	/* dir read */
+	.opendir     = pump_opendir,
+	.readdir     = pump_readdir,
+	.readdirp    = pump_readdirp,
+
+	/* dir write */
+	.create      = pump_create,
+	.mknod       = pump_mknod,
+	.mkdir       = pump_mkdir,
+	.unlink      = pump_unlink,
+	.rmdir       = pump_rmdir,
+	.link        = pump_link,
+	.symlink     = pump_symlink,
+	.rename      = pump_rename,
+};
+
+struct xlator_dumpops dumpops = {
+        .priv       = afr_priv_dump,
+};
+
+
+struct xlator_cbks cbks = {
+	.release     = pump_release,
+	.releasedir  = pump_releasedir,
+        .forget      = pump_forget,
+};
+
+struct volume_options options[] = {
+	{ .key  = {NULL} },
+};
diff --git a/xlators/cluster/afr/src/pump.h b/xlators/cluster/afr/src/pump.h
new file mode 100644
index 000000000..bc4c31a78
--- /dev/null
+++ b/xlators/cluster/afr/src/pump.h
@@ -0,0 +1,78 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __PUMP_H__
+#define __PUMP_H__
+
+#include "syncop.h"
+
+/* FIXME: Needs to be defined in a common file */
+#define CLIENT_CMD_CONNECT "trusted.glusterfs.client-connect"
+#define CLIENT_CMD_DISCONNECT "trusted.glusterfs.client-disconnect"
+
+#define PUMP_SOURCE_COMPLETE "trusted.glusterfs.pump-source-complete"
+#define PUMP_SINK_COMPLETE "trusted.glusterfs.pump-sink-complete"
+
+#define PUMP_PATH "trusted.glusterfs.pump-path"
+
+#define PUMP_SOURCE_CHILD(xl) (xl->children->xlator)
+#define PUMP_SINK_CHILD(xl) (xl->children->next->xlator)
+
+typedef enum {
+        PUMP_STATE_RUNNING,             /* Pump is running and migrating files */
+        PUMP_STATE_RESUME,              /* Pump is resuming from a previous pause */
+        PUMP_STATE_PAUSE,               /* Pump is paused */
+        PUMP_STATE_ABORT,               /* Pump is aborted */
+        PUMP_STATE_COMMIT,              /* Pump is commited */
+} pump_state_t;
+
+typedef struct _pump_private {
+	struct syncenv *env;            /* The env pointer to the pump synctask */
+        char *resume_path;              /* path to resume from the last pause */
+        gf_lock_t resume_path_lock;     /* Synchronize resume_path changes */
+        gf_lock_t pump_state_lock;      /* Synchronize pump_state changes */
+        pump_state_t pump_state;        /* State of pump */
+        char current_file[PATH_MAX];    /* Current file being pumped */
+        uint64_t number_files_pumped;   /* Number of files pumped */
+        gf_boolean_t pump_finished;     /* Boolean to indicate pump termination */
+        char pump_start_pending;        /* Boolean to mark start pending until
+                                           CHILD_UP */
+        call_stub_t *cleaner;
+} pump_private_t;
+
+void
+build_root_loc (inode_t *inode, loc_t *loc);
+int pump_start (call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+pump_command_start (xlator_t *this, dict_t *dict);
+
+int
+pump_execute_start (call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+pump_command_pause (xlator_t *this, dict_t *dict);
+
+int
+pump_execute_pause (call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+pump_command_abort (xlator_t *this, dict_t *dict);
+
+int
+pump_execute_abort (call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+pump_command_status (xlator_t *this, dict_t *dict);
+
+int
+pump_execute_status (call_frame_t *frame, xlator_t *this);
+
+#endif /* __PUMP_H__ */
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index b7d07d137..174bea841 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -1,30 +1,38 @@
 
-xlator_LTLIBRARIES = dht.la nufa.la
+xlator_LTLIBRARIES = dht.la nufa.la switch.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
+dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c dht-rebalance.c \
+	dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c \
+	dht-common.c dht-inode-write.c dht-inode-read.c dht-shared.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c \
-		dht-selfheal.c dht-rename.c dht-hashfn.c dht-hashfn-tea.c
-
-dht_la_SOURCES = $(dht_common_source) dht.c 
+dht_la_SOURCES = $(dht_common_source) dht.c
 
 nufa_la_SOURCES = $(dht_common_source) nufa.c
+switch_la_SOURCES = $(dht_common_source) switch.c
 
-dht_la_LDFLAGS = -module -avoidversion
+dht_la_LDFLAGS = -module -avoid-version
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoidversion
+nufa_la_LDFLAGS = -module -avoid-version
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = dht-common.h dht-common.c
+switch_la_LDFLAGS = -module -avoid-version
+switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = dht-common.h dht-mem-types.h \
+	$(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/lib/src
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
-CLEANFILES = 
+CLEANFILES =
 
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/distribute.so
 
 install-data-hook:
-	ln -sf dht.so $(DESTDIR)$(xlatordir)/distribute.so
\ No newline at end of file
+	ln -sf dht.so $(DESTDIR)$(xlatordir)/distribute.so
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index f121f9f47..8f61339e6 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -27,9 +18,87 @@
 
 #include "glusterfs.h"
 #include "xlator.h"
+#include "libxlator.h"
 #include "dht-common.h"
 #include "defaults.h"
+#include "byte-order.h"
+#include "glusterfs-acl.h"
+
+#include <sys/time.h>
+#include <libgen.h>
+
+int
+dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
+{
+        dict_t  *dst  = NULL;
+        int64_t *ptr  = 0, *size = NULL;
+        int32_t  ret  = -1;
+        data_t  *dict_data = NULL;
+
+        dst = data;
+
+        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
+                ret = dict_get_bin (dst, key, (void **)&size);
+                if (ret < 0) {
+                        size = GF_CALLOC (1, sizeof (int64_t),
+                                          gf_common_mt_char);
+                        if (size == NULL) {
+                                gf_log ("dht", GF_LOG_WARNING,
+                                        "memory allocation failed");
+                                return -1;
+                        }
+                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
+                        if (ret < 0) {
+                                gf_log ("dht", GF_LOG_WARNING,
+                                        "dht aggregate dict set failed");
+                                GF_FREE (size);
+                                return -1;
+                        }
+                }
+
+                ptr = data_to_bin (value);
+                if (ptr == NULL) {
+                        gf_log ("dht", GF_LOG_WARNING, "data to bin failed");
+                        return -1;
+                }
 
+                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+                ret = gf_get_min_stime (THIS, dst, key, value);
+                if (ret < 0)
+                        return ret;
+        } else {
+                /* compare user xattrs only */
+                if (!strncmp (key, "user.", strlen ("user."))) {
+                        ret = dict_lookup (dst, key, &dict_data);
+                        if (!ret && dict_data && value) {
+                                ret = is_data_equal (dict_data, value);
+                                if (!ret)
+                                        gf_log ("dht", GF_LOG_DEBUG,
+                                                "xattr mismatch for %s", key);
+                        }
+                }
+                ret = dict_set (dst, key, value);
+                if (ret)
+                        gf_log ("dht", GF_LOG_WARNING, "xattr dict set failed");
+        }
+
+        return 0;
+}
+
+
+void
+dht_aggregate_xattr (dict_t *dst, dict_t *src)
+{
+        if ((dst == NULL) || (src == NULL)) {
+                goto out;
+        }
+
+        dict_foreach (src, dht_aggregate, dst);
+out:
+        return;
+}
 
 /* TODO:
    - use volumename in xattr instead of "dht"
@@ -38,99 +107,372 @@
    - complete linkfile selfheal
 */
 
+
 int
 dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
-			 xlator_t *this,
-			 int op_ret, int op_errno)
+                         xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	dht_layout_t *layout = NULL;
-	int           ret = 0;
+        dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+	int           ret = -1;
 
-	local = frame->local;
-	ret = op_ret;
-
-	if (ret == 0) {
-		layout = local->selfheal.layout;
-		ret = inode_ctx_put (local->inode, this, (uint64_t)(long)layout);
-
-		if (ret == 0)
-			local->selfheal.layout = NULL;
-		
-		if (local->st_ino) {
-			local->stbuf.st_ino = local->st_ino;
-		} else {
-			gf_log (this->name, GF_LOG_WARNING,
-				"could not find hashed subvolume for %s",
-				local->loc.path);
-		}
-	}
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+
+        local = frame->local;
+        ret = op_ret;
 
-	DHT_STACK_UNWIND (frame, ret, local->op_errno, local->inode,
-			  &local->stbuf, local->xattr);
+        FRAME_SU_UNDO (frame, dht_local_t);
 
-	return 0;
+        if (ret == 0) {
+                layout = local->selfheal.layout;
+                ret = dht_layout_set (this, local->inode, layout);
+        }
+
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           &local->postparent, 1);
+        }
+
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+
+        DHT_STACK_UNWIND (lookup, frame, ret, local->op_errno, local->inode,
+                          &local->stbuf, local->xattr, &local->postparent);
+
+out:
+        return ret;
 }
 
 
 int
-dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct stat *stbuf, dict_t *xattr)
+dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
 {
-	dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev          = NULL;
-	dht_layout_t *layout        = NULL;
-	int           ret           = 0;
-	int           is_dir        = 0;
+        dht_local_t     *local = NULL;
+        call_frame_t    *main_frame = NULL;
+        int              op_errno = 0;
+        int              ret = -1;
+        dht_layout_t    *layout = NULL;
+        dht_conf_t      *conf = NULL;
+
+        local = discover_frame->local;
+        layout = local->layout;
+        conf = this->private;
+
+        LOCK(&discover_frame->lock);
+        {
+                main_frame = local->main_frame;
+                local->main_frame = NULL;
+        }
+        UNLOCK(&discover_frame->lock);
+
+        if (!main_frame)
+                return 0;
+
+        if (local->file_count && local->dir_count) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "path %s exists as a file on one subvolume "
+                        "and directory on another. "
+                        "Please fix it manually",
+                        local->loc.path);
+                op_errno = EIO;
+                goto out;
+        }
+
+        if (local->cached_subvol) {
+                ret = dht_layout_preset (this, local->cached_subvol,
+                                         local->inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set layout for subvolume %s",
+                                local->cached_subvol ? local->cached_subvol->name : "<nil>");
+                        op_errno = EINVAL;
+                        goto out;
+                }
+        } else {
+                ret = dht_layout_normalize (this, &local->loc, layout);
+                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+                        /* either the layout is incorrect or the directory is
+                         * not found even in one subvolume.
+                         */
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "normalizing failed on %s "
+                                "(overlaps/holes present: %s, "
+                                "ENOENT errors: %d)", local->loc.path,
+                                (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
+                        if ((ret > 0) && (ret == conf->subvolume_cnt)) {
+                                op_errno = ESTALE;
+                                goto out;
+                        }
+                }
+
+                if (local->inode)
+                        dht_layout_set (this, local->inode, layout);
+        }
+
+        DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
+                          local->inode, &local->stbuf, local->xattr,
+                          &local->postparent);
+        return 0;
+out:
+        DHT_STACK_UNWIND (lookup, main_frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
+
+        return ret;
+}
+
+
+int
+dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno,
+                  inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                  struct iatt *postparent)
+{
+        dht_local_t  *local                   = NULL;
+        int           this_call_cnt           = 0;
+        call_frame_t *prev                    = NULL;
+        dht_layout_t *layout                  = NULL;
+        int           ret                     = -1;
+        int           is_dir                  = 0;
+        int           is_linkfile             = 0;
+        int           attempt_unwind          = 0;
+        dht_conf_t   *conf                    = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
 
-	conf  = this->private;
         local = frame->local;
         prev  = cookie;
+        conf = this->private;
+
+        layout = local->layout;
+
+        /* Check if the gfid is different for file from other node */
+        if (!op_ret && uuid_compare (local->gfid, stbuf->ia_gfid)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: gfid different on %s",
+                        local->loc.path, prev->this->name);
+        }
 
-	layout = local->layout;
 
         LOCK (&frame->lock);
         {
                 /* TODO: assert equal mode on stbuf->st_mode and
-		   local->stbuf->st_mode
+                   local->stbuf->st_mode
 
-		   else mkdir/chmod/chown and fix
-		*/
-		/* TODO: assert equal hash type in xattr, local->xattr */
+                   else mkdir/chmod/chown and fix
+                */
+                ret = dht_layout_merge (this, layout, prev->this,
+                                        op_ret, op_errno, xattr);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to merge layouts", local->loc.path);
 
-		/* TODO: always ensure same subvolume is in layout->list[0] */
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "lookup of %s on %s returned error (%s)",
+                                local->loc.path, prev->this->name,
+                                strerror (op_errno));
 
-		ret = dht_layout_merge (this, layout, prev->this,
-					op_ret, op_errno, xattr);
+                        goto unlock;
+                }
 
-		if (op_ret == -1) {
-			local->op_errno = ENOENT;
-			gf_log (this->name, GF_LOG_WARNING,
-				"lookup of %s on %s returned error (%s)",
-				local->loc.path, prev->this->name,
-				strerror (op_errno));
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
+                is_dir = check_is_dir (inode, stbuf, xattr);
+
+                if (is_dir) {
+                        local->dir_count ++;
+                } else {
+                        local->file_count ++;
+
+                        if (!is_linkfile) {
+                                /* real file */
+                                local->cached_subvol = prev->this;
+                                attempt_unwind = 1;
+                        } else {
+                                goto unlock;
+                        }
+                }
 
-			goto unlock;
-		}
+                local->op_ret = 0;
+
+                if (local->xattr == NULL) {
+                        local->xattr = dict_ref (xattr);
+                } else {
+                        dht_aggregate_xattr (local->xattr, xattr);
+                }
+
+                if (local->inode == NULL)
+                        local->inode = inode_ref (inode);
+
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
+        }
+unlock:
+        UNLOCK (&frame->lock);
+out:
+        this_call_cnt = dht_frame_return (frame);
+
+        if (is_last_call (this_call_cnt) || attempt_unwind) {
+                dht_discover_complete (this, frame);
+        }
+
+        if (is_last_call (this_call_cnt))
+                DHT_STACK_DESTROY (frame);
+
+        return 0;
+}
+
+
+int
+dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+        int          ret;
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf = NULL;
+        int          call_cnt = 0;
+        int          op_errno = EINVAL;
+        int          i = 0;
+        call_frame_t *discover_frame = NULL;
+
+        conf = this->private;
+        local = frame->local;
 
- 		is_dir = check_is_dir (inode, stbuf, xattr);
- 		if (!is_dir) 
- 			goto unlock;
+        ret = dict_set_uint32 (local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->xattr_name);
 
- 		local->op_ret = 0;
- 		if (local->xattr == NULL)
- 			local->xattr = dict_ref (xattr);
- 		if (local->inode == NULL)
- 			local->inode = inode_ref (inode);
+        ret = dict_set_uint32 (local->xattr_req, conf->link_xattr_name, 256);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->link_xattr_name);
 
-		dht_stat_merge (this, &local->stbuf, stbuf, prev->this);
+        call_cnt        = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-		if (prev->this == local->hashed_subvol)
-			local->st_ino = local->stbuf.st_ino;
+        local->layout = dht_layout_new (this, conf->subvolume_cnt);
 
+        if (!local->layout) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        uuid_copy (local->gfid, loc->gfid);
+
+        discover_frame = copy_frame (frame);
+        if (!discover_frame) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        discover_frame->local = local;
+        frame->local = NULL;
+        local->main_frame = frame;
+
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (discover_frame, dht_discover_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->lookup,
+                            &local->loc, local->xattr_req);
+        }
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
+
+        return 0;
+}
+
+
+int
+dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno,
+                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                    struct iatt *postparent)
+{
+        dht_local_t  *local                   = NULL;
+        int           this_call_cnt           = 0;
+        call_frame_t *prev                    = NULL;
+        dht_layout_t *layout                  = NULL;
+        int           ret                     = -1;
+        int           is_dir                  = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev  = cookie;
+
+        layout = local->layout;
+
+        if (!op_ret && uuid_is_null (local->gfid))
+                memcpy (local->gfid, stbuf->ia_gfid, 16);
+
+        /* Check if the gfid is different for file from other node */
+        if (!op_ret && uuid_compare (local->gfid, stbuf->ia_gfid)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: gfid different on %s",
+                        local->loc.path, prev->this->name);
+        }
+
+        LOCK (&frame->lock);
+        {
+                /* TODO: assert equal mode on stbuf->st_mode and
+                   local->stbuf->st_mode
+
+                   else mkdir/chmod/chown and fix
+                */
+                ret = dht_layout_merge (this, layout, prev->this,
+                                        op_ret, op_errno, xattr);
+
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "lookup of %s on %s returned error (%s)",
+                                local->loc.path, prev->this->name,
+                                strerror (op_errno));
+
+                        goto unlock;
+                }
+
+                is_dir = check_is_dir (inode, stbuf, xattr);
+                if (!is_dir) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "lookup of %s on %s returned non dir 0%o",
+                                local->loc.path, prev->this->name,
+                                stbuf->ia_type);
+                        local->need_selfheal = 1;
+                        goto unlock;
+                }
+
+                local->op_ret = 0;
+                if (local->xattr == NULL) {
+                        local->xattr = dict_ref (xattr);
+                } else {
+                        dht_aggregate_xattr (local->xattr, xattr);
+                }
+
+                if (local->inode == NULL)
+                        local->inode = inode_ref (inode);
+
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
         }
 unlock:
         UNLOCK (&frame->lock);
@@ -139,372 +481,779 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
 
         if (is_last_call (this_call_cnt)) {
-		if (local->op_ret == 0) {
-			ret = dht_layout_normalize (this, &local->loc, layout);
+                if (local->need_selfheal) {
+                        local->need_selfheal = 0;
+                        dht_lookup_everywhere (frame, this, &local->loc);
+                        return 0;
+                }
 
-			local->layout = NULL;
+                if (local->op_ret == 0) {
+                        ret = dht_layout_normalize (this, &local->loc, layout);
 
-			if (ret != 0) {
-				layout->gen = conf->gen;
+                        if (ret != 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "fixing assignment on %s",
+                                        local->loc.path);
+                                goto selfheal;
+                        }
 
-				gf_log (this->name, GF_LOG_WARNING,
-					"fixing assignment on %s",
-					local->loc.path);
-				goto selfheal;
-			}
-			
-			inode_ctx_put (local->inode, this, (uint64_t)(long)layout);
-			
-			if (local->st_ino) {
-				local->stbuf.st_ino = local->st_ino;
-			} else {
-				gf_log (this->name, GF_LOG_WARNING,
-					"could not find hashed subvolume for %s",
-					local->loc.path);
-			}
-		}
+                        dht_layout_set (this, local->inode, layout);
+                }
+
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
-		DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  local->inode, &local->stbuf, local->xattr);
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+                                  local->inode, &local->stbuf, local->xattr,
+                                  &local->postparent);
         }
 
-	return 0;
+        return 0;
 
 selfheal:
-	ret = dht_selfheal_directory (frame, dht_lookup_selfheal_cbk,
-				      &local->loc, layout);
-
-	return 0;
+        FRAME_SU_DO (frame, dht_local_t);
+        uuid_copy (local->loc.gfid, local->gfid);
+        ret = dht_selfheal_directory (frame, dht_lookup_selfheal_cbk,
+                                      &local->loc, layout);
+out:
+        return ret;
 }
 
 int
 dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int op_ret, int op_errno,
-                    inode_t *inode, struct stat *stbuf, dict_t *xattr)
+                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                    struct iatt *postparent)
 {
         dht_local_t  *local         = NULL;
         int           this_call_cnt = 0;
         call_frame_t *prev          = NULL;
-	dht_layout_t *layout        = NULL;
-	int           ret  = -1;
-	int           is_dir = 0;
-	int           is_linkfile = 0;
+        dht_layout_t *layout        = NULL;
+        dht_conf_t   *conf          = NULL;
+        int           ret  = -1;
+        int           is_dir = 0;
+        int           is_linkfile = 0;
+        call_frame_t *copy          = NULL;
+        dht_local_t  *copy_local    = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, err);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, err);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, err);
 
         local = frame->local;
         prev  = cookie;
+        conf = this->private;
+        if (!conf)
+                goto out;
 
         LOCK (&frame->lock);
         {
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-
-			if (op_errno != ENOTCONN && op_errno != ENOENT) {
-				gf_log (this->name, GF_LOG_WARNING,
-					"subvolume %s returned -1 (%s)",
-					prev->this->name, strerror (op_errno));
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+
+                        if ((op_errno != ENOTCONN)
+                            && (op_errno != ENOENT)
+                            && (op_errno != ESTALE)) {
+				gf_log (this->name, GF_LOG_INFO,
+					"subvolume %s for %s returned -1 (%s)",
+					prev->this->name, local->loc.path,
+                                        strerror (op_errno));
 			}
+                        if (op_errno == ESTALE) {
+                                /* propagate the ESTALE to parent.
+                                 * setting local->return_estale would send
+                                 * ESTALE to parent. */
+                                local->return_estale = 1;
+                        }
+
+                        /* if it is ENOENT, we may have to do a
+                         * 'lookup_everywhere()' to make sure
+                         * the file is not migrated */
+                        if (op_errno == ENOENT) {
+                                if (IA_ISREG (local->loc.inode->ia_type)) {
+                                        local->need_lookup_everywhere = 1;
+                                }
+                        }
+                        goto unlock;
+                }
 
-			goto unlock;
-		}
+                if (stbuf->ia_type != local->inode->ia_type) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "mismatching filetypes 0%o v/s 0%o for %s",
+                                (stbuf->ia_type), (local->inode->ia_type),
+                                local->loc.path);
 
-		if (S_IFMT & (stbuf->st_mode ^ local->inode->st_mode)) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"mismatching filetypes 0%o v/s 0%o for %s",
-				(stbuf->st_mode & S_IFMT),
-				(local->inode->st_mode & S_IFMT),
-				local->loc.path);
+                        local->op_ret = -1;
+                        local->op_errno = EINVAL;
 
-			local->op_ret = -1;
-			local->op_errno = EINVAL;
+                        goto unlock;
+                }
 
-			goto unlock;
-		}
+                layout = local->layout;
 
-		layout = dht_layout_get (this, inode);
-		
-		is_dir = check_is_dir (inode, stbuf, xattr);
-		is_linkfile = check_is_linkfile (inode, stbuf, xattr);
-		
-		if (is_linkfile) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"linkfile found in revalidate for %s",
-				local->loc.path);
-			local->layout_mismatch = 1;
-
-			goto unlock;
-		}
+                is_dir = check_is_dir (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
 
-		if (is_dir) {
-			ret = dht_layout_dir_mismatch (this, layout,
-						       prev->this, &local->loc,
-						       xattr);
-			if (ret != 0) {
-				gf_log (this->name, GF_LOG_WARNING,
-					"mismatching layouts for %s", 
-					local->loc.path);
-			
-				local->layout_mismatch = 1;
-
-				goto unlock;
-			}
-		} 
-		
-		dht_stat_merge (this, &local->stbuf, stbuf, prev->this);
-		
-		local->op_ret = 0;
-		local->stbuf.st_ino = local->st_ino;
-
-		if (!local->xattr)
-			local->xattr = dict_ref (xattr);
-	}
-unlock:
-	UNLOCK (&frame->lock);
+                if (is_linkfile) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "linkfile found in revalidate for %s",
+                                local->loc.path);
+                        local->return_estale = 1;
+
+                        goto unlock;
+                }
 
+                if (is_dir) {
+                        ret = dht_dir_has_layout (xattr, conf->xattr_name);
+                        if (ret >= 0) {
+                                if (is_greater_time(local->stbuf.ia_ctime,
+                                                    local->stbuf.ia_ctime_nsec,
+                                                    stbuf->ia_ctime,
+                                                    stbuf->ia_ctime_nsec)) {
+                                        local->prebuf.ia_gid = stbuf->ia_gid;
+                                        local->prebuf.ia_uid = stbuf->ia_uid;
+                                }
+                        }
+                        if (local->stbuf.ia_type != IA_INVAL)
+                        {
+                                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                                    (local->stbuf.ia_uid != stbuf->ia_uid)) {
+                                        local->need_selfheal = 1;
+                                }
+                        }
+                        ret = dht_layout_dir_mismatch (this, layout,
+                                                       prev->this, &local->loc,
+                                                       xattr);
+                        if (ret != 0) {
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "mismatching layouts for %s",
+                                        local->loc.path);
+
+                                local->layout_mismatch = 1;
+
+                                goto unlock;
+                        }
+                }
+
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
+
+                local->op_ret = 0;
+
+                if (!local->xattr) {
+                        local->xattr = dict_ref (xattr);
+                } else if (is_dir) {
+                        dht_aggregate_xattr (local->xattr, xattr);
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+out:
         this_call_cnt = dht_frame_return (frame);
 
         if (is_last_call (this_call_cnt)) {
-		if (!S_ISDIR (local->stbuf.st_mode)
-		    && (local->hashed_subvol != local->cached_subvol)
-		    && (local->stbuf.st_nlink == 1))
-			local->stbuf.st_mode |= S_ISVTX;
-		
+                if (!IA_ISDIR (local->stbuf.ia_type)
+                    && (local->hashed_subvol != local->cached_subvol)
+                    && (local->stbuf.ia_nlink == 1)
+                    && (conf && conf->unhashed_sticky_bit)) {
+                        local->stbuf.ia_prot.sticky = 1;
+                }
+                if (local->need_selfheal) {
+                        local->need_selfheal = 0;
+                        uuid_copy (local->gfid, local->stbuf.ia_gfid);
+                        local->stbuf.ia_gid = local->prebuf.ia_gid;
+                        local->stbuf.ia_uid = local->prebuf.ia_uid;
+                        copy = create_frame (this, this->ctx->pool);
+                        if (copy) {
+                                copy_local = dht_local_init (copy, &local->loc,
+                                                             NULL, 0);
+                                if (!copy_local)
+                                        goto cont;
+                                copy_local->stbuf = local->stbuf;
+                                copy->local = copy_local;
+                                FRAME_SU_DO (copy, dht_local_t);
+                                ret = synctask_new (this->ctx->env,
+                                                    dht_dir_attr_heal,
+                                                    dht_dir_attr_heal_done,
+                                                    copy, copy);
+                        }
+                }
+cont:
 		if (local->layout_mismatch) {
-			local->op_ret = -1;
-			local->op_errno = ESTALE;
-		}
-			
-		DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  local->inode, &local->stbuf, local->xattr);
-	}
+                        /* Found layout mismatch in the directory, need to
+                           fix this in the inode context */
+                        dht_layout_unref (this, local->layout);
+                        local->layout = NULL;
+                        dht_lookup_directory (frame, this, &local->loc);
+                        return 0;
+                }
 
-        return 0;
+                if (local->need_lookup_everywhere) {
+                        /* As the current layout gave ENOENT error, we would
+                           need a new layout */
+                        dht_layout_unref (this, local->layout);
+                        local->layout = NULL;
+
+                        /* We know that current cached subvol is no more
+                           valid, get the new one */
+                        local->cached_subvol = NULL;
+                        dht_lookup_everywhere (frame, this, &local->loc);
+                        return 0;
+                }
+                if (local->return_estale) {
+                        local->op_ret = -1;
+                        local->op_errno = ESTALE;
+                }
+
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+                                  local->inode, &local->stbuf, local->xattr,
+                                  &local->postparent);
+        }
+
+err:
+        return ret;
 }
 
 
 int
 dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-				xlator_t *this,
-				int32_t op_ret, int32_t op_errno,
-				inode_t *inode, struct stat *stbuf)
+                                xlator_t *this,
+                                int32_t op_ret, int32_t op_errno,
+                                inode_t *inode, struct iatt *stbuf,
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	dht_layout_t *layout = NULL;
-	xlator_t     *cached_subvol = NULL;
+        dht_local_t  *local = NULL;
+        xlator_t     *cached_subvol = NULL;
+        dht_conf_t   *conf = NULL;
+        int           ret = -1;
 
-	local = frame->local;
-	cached_subvol = local->cached_subvol;
-
-	layout = dht_layout_for_subvol (this, local->cached_subvol);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no pre-set layout for subvolume %s",
-			cached_subvol ? cached_subvol->name : "<nil>");
-		local->op_ret = -1;
-		local->op_errno = EINVAL;
-		goto unwind;
-	}
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
 
-	inode_ctx_put (local->inode, this, (uint64_t)(long)layout);
-	local->op_ret = 0;
-	if (local->stbuf.st_nlink == 1)
-		local->stbuf.st_mode |= S_ISVTX;
+        local = frame->local;
+        cached_subvol = local->cached_subvol;
+        conf = this->private;
+
+        ret = dht_layout_preset (this, local->cached_subvol, inode);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to set layout for subvolume %s",
+                        cached_subvol ? cached_subvol->name : "<nil>");
+                local->op_ret = -1;
+                local->op_errno = EINVAL;
+                goto unwind;
+        }
+
+        local->op_ret = 0;
+        if ((local->stbuf.ia_nlink == 1)
+            && (conf && conf->unhashed_sticky_bit)) {
+                local->stbuf.ia_prot.sticky = 1;
+        }
+
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
 unwind:
-	DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			  local->inode, &local->stbuf, local->xattr);
-	return 0;
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
+
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+                          local->inode, &local->stbuf, local->xattr,
+                          &local->postparent);
+out:
+        return ret;
+}
+
+
+int
+dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
+{
+        int           ret = 0;
+        dht_local_t  *local = NULL;
+        xlator_t     *hashed_subvol = NULL;
+        xlator_t     *cached_subvol = NULL;
+        dht_layout_t *layout = NULL;
+
+        local = frame->local;
+        hashed_subvol = local->hashed_subvol;
+        cached_subvol = local->cached_subvol;
+
+        if (local->file_count && local->dir_count) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "path %s exists as a file on one subvolume "
+                        "and directory on another. "
+                        "Please fix it manually",
+                        local->loc.path);
+                DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL,
+                                  NULL);
+                return 0;
+        }
+
+        if (local->dir_count) {
+                dht_lookup_directory (frame, this, &local->loc);
+                return 0;
+        }
+
+        if (!cached_subvol) {
+                DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                  NULL);
+                return 0;
+        }
+
+        if (local->need_lookup_everywhere) {
+                if (uuid_compare (local->gfid, local->inode->gfid)) {
+                        /* GFID different, return error */
+                        DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL,
+                                          NULL, NULL, NULL);
+                        return 0;
+                }
+                local->op_ret = 0;
+                local->op_errno = 0;
+                layout = dht_layout_for_subvol (this, cached_subvol);
+                if (!layout) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%s: no pre-set layout for subvolume %s",
+                                local->loc.path, (cached_subvol ?
+                                                  cached_subvol->name :
+                                                  "<nil>"));
+                }
+
+                ret = dht_layout_set (this, local->inode, layout);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%s: failed to set layout for subvol %s",
+                                local->loc.path, (cached_subvol ?
+                                                  cached_subvol->name :
+                                                  "<nil>"));
+                }
+
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
+                                  local->op_errno, local->inode,
+                                  &local->stbuf, local->xattr,
+                                  &local->postparent);
+                return 0;
+        }
+
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "cannot create linkfile file for %s on %s: "
+                        "hashed subvolume cannot be found.",
+                        local->loc.path, cached_subvol->name);
+
+                local->op_ret = 0;
+                local->op_errno = 0;
+
+                ret = dht_layout_preset (frame->this, cached_subvol,
+                                         local->inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "failed to set layout for subvol %s",
+                                cached_subvol ? cached_subvol->name :
+                                "<nil>");
+                        local->op_ret = -1;
+                        local->op_errno = EINVAL;
+                }
+
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
+                                  local->op_errno, local->inode,
+                                  &local->stbuf, local->xattr,
+                                  &local->postparent);
+                return 0;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "linking file %s existing on %s to %s (hash)",
+                local->loc.path, cached_subvol->name,
+                hashed_subvol->name);
+
+        ret = dht_linkfile_create (frame,
+                                   dht_lookup_linkfile_create_cbk, this,
+                                   cached_subvol, hashed_subvol, &local->loc);
+
+        return ret;
+}
+
+
+int
+dht_lookup_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno,
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
+{
+        int  this_call_cnt = 0;
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                dht_lookup_everywhere_done (frame, this);
+        }
+
+        return 0;
 }
 
 
 int
 dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno,
-			   inode_t *inode, struct stat *buf, dict_t *xattr)
+                           int32_t op_ret, int32_t op_errno,
+                           inode_t *inode, struct iatt *buf, dict_t *xattr,
+                           struct iatt *postparent)
 {
-	dht_conf_t   *conf          = NULL;
         dht_local_t  *local         = NULL;
         int           this_call_cnt = 0;
         call_frame_t *prev          = NULL;
-	int           is_linkfile   = 0;
-	int           is_dir        = 0;
-	xlator_t     *subvol        = NULL;
-	loc_t        *loc           = NULL;
-	xlator_t     *link_subvol   = NULL;
-	xlator_t     *hashed_subvol = NULL;
-	xlator_t     *cached_subvol = NULL;
-
-	conf   = this->private;
-
-	local  = frame->local;
-	loc    = &local->loc;
-
-	prev   = cookie;
-	subvol = prev->this;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			if (op_errno != ENOENT)
-				local->op_errno = op_errno;
-			goto unlock;
-		}
+        int           is_linkfile   = 0;
+        int           is_dir        = 0;
+        xlator_t     *subvol        = NULL;
+        loc_t        *loc           = NULL;
+        xlator_t     *link_subvol   = NULL;
+        int           ret           = -1;
+        int32_t       fd_count      = 0;
+        dht_conf_t   *conf          = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+
+        local  = frame->local;
+        loc    = &local->loc;
+        conf   = this->private;
 
-		is_linkfile = check_is_linkfile (inode, buf, xattr);
-		is_dir = check_is_dir (inode, buf, xattr);
-
-		if (is_linkfile) {
-			link_subvol = dht_linkfile_subvol (this, inode, buf,
-							   xattr);
-			gf_log (this->name, GF_LOG_DEBUG,
-				"found on %s linkfile %s (-> %s)",
-				subvol->name, loc->path,
-				link_subvol ? link_subvol->name : "''");
-			goto unlock;
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"found on %s file %s",
-				subvol->name, loc->path);
-		}
+        prev   = cookie;
+        subvol = prev->this;
 
-		if (!local->cached_subvol) {
-			/* found one file */
-			dht_stat_merge (this, &local->stbuf, buf, subvol);
-			local->xattr = dict_ref (xattr);
-			local->cached_subvol = subvol;
-		} else {
-			gf_log (this->name, GF_LOG_WARNING,
-				"multiple subvolumes (%s and %s atleast) have "
-				"file %s", local->cached_subvol->name,
-				subvol->name, local->loc.path);
-		}
-	}
-unlock:
-	UNLOCK (&frame->lock);
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        if (op_errno != ENOENT)
+                                local->op_errno = op_errno;
+                        goto unlock;
+                }
 
-	if (is_linkfile) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"deleting stale linkfile %s on %s",
-			loc->path, subvol->name);
-		dht_linkfile_unlink (frame, this, subvol, loc);
-	}
+                if (uuid_is_null (local->gfid))
+                        uuid_copy (local->gfid, buf->ia_gfid);
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		hashed_subvol = local->hashed_subvol;
-		cached_subvol = local->cached_subvol;
+                if (uuid_compare (local->gfid, buf->ia_gfid)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: gfid differs on subvolume %s",
+                                loc->path, prev->this->name);
+                }
 
-		if (!cached_subvol) {
-			DHT_STACK_UNWIND (frame, -1, ENOENT, NULL, NULL, NULL);
-			return 0;
-		}
+                is_linkfile = check_is_linkfile (inode, buf, xattr,
+                                                 conf->link_xattr_name);
+                is_dir = check_is_dir (inode, buf, xattr);
+
+                if (is_linkfile) {
+                        link_subvol = dht_linkfile_subvol (this, inode, buf,
+                                                           xattr);
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "found on %s linkfile %s (-> %s)",
+                                subvol->name, loc->path,
+                                link_subvol ? link_subvol->name : "''");
+                        goto unlock;
+                }
 
-		gf_log (this->name, GF_LOG_WARNING,
-			"linking file %s existing on %s to %s (hash)",
-			loc->path, cached_subvol->name, hashed_subvol->name);
+                /* non linkfile GFID takes precedence */
+                uuid_copy (local->gfid, buf->ia_gfid);
+
+                if (is_dir) {
+                        local->dir_count++;
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "found on %s directory %s",
+                                subvol->name, loc->path);
+                } else {
+                        local->file_count++;
+
+                        if (!local->cached_subvol) {
+                                /* found one file */
+                                dht_iatt_merge (this, &local->stbuf, buf,
+                                                subvol);
+                                local->xattr = dict_ref (xattr);
+                                local->cached_subvol = subvol;
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "found on %s file %s",
+                                        subvol->name, loc->path);
+
+                                dht_iatt_merge (this, &local->postparent,
+                                                postparent, subvol);
+                        } else {
+                                /* This is where we need 'rename' both entries logic */
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "multiple subvolumes (%s and %s) have "
+                                        "file %s (preferably rename the file "
+                                        "in the backend, and do a fresh lookup)",
+                                        local->cached_subvol->name,
+                                        subvol->name, local->loc.path);
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
 
-		dht_linkfile_create (frame, dht_lookup_linkfile_create_cbk,
-				     cached_subvol, hashed_subvol, loc);
-	}
+        if (is_linkfile) {
+                ret = dict_get_int32 (xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
+                /* Delete the linkfile only if there are no open fds on it.
+                   if there is a open-fd, it may be in migration */
+                if (!ret && (fd_count == 0)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "deleting stale linkfile %s on %s",
+                                loc->path, subvol->name);
+                        STACK_WIND (frame, dht_lookup_unlink_cbk,
+                                    subvol, subvol->fops->unlink, loc, 0, NULL);
+                        return 0;
+                }
+        }
 
-	return 0;
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                dht_lookup_everywhere_done (frame, this);
+        }
+
+out:
+        return ret;
 }
 
 
 int
 dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-	dht_conf_t     *conf = NULL;
-	dht_local_t    *local = NULL;
-	int             i = 0;
-	int             call_cnt = 0;
-
-	conf = this->private;
-	local = frame->local;
+        dht_conf_t     *conf = NULL;
+        dht_local_t    *local = NULL;
+        int             i = 0;
+        int             call_cnt = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+        GF_VALIDATE_OR_GOTO ("dht", loc, out);
+
+        conf = this->private;
+        local = frame->local;
 
-	call_cnt = conf->subvolume_cnt;
-	local->call_cnt = call_cnt;
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-	if (!local->inode)
-		local->inode = inode_ref (loc->inode);
+        if (!local->inode)
+                local->inode = inode_ref (loc->inode);
 
-	for (i = 0; i < call_cnt; i++) {
-		STACK_WIND (frame, dht_lookup_everywhere_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->lookup,
-			    loc, local->xattr_req);
-	}
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_lookup_everywhere_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->lookup,
+                            loc, local->xattr_req);
+        }
 
-	return 0;
+        return 0;
+out:
+        DHT_STACK_UNWIND (lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+err:
+        return -1;
 }
 
 
 int
 dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                          xlator_t *this, int op_ret, int op_errno,
-                         inode_t *inode, struct stat *stbuf, dict_t *xattr)
+                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                         struct iatt *postparent)
 {
-        call_frame_t *prev = NULL;
-	dht_local_t  *local = NULL;
-	dht_layout_t *layout = NULL;
-	xlator_t     *subvol = NULL;
-	loc_t        *loc = NULL;
+        call_frame_t *prev          = NULL;
+        dht_local_t  *local         = NULL;
+        xlator_t     *subvol        = NULL;
+        loc_t        *loc           = NULL;
+        dht_conf_t   *conf          = NULL;
+        int           ret           = 0;
 
-        prev   = cookie;
-	subvol = prev->this;
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, unwind);
 
-	local  = frame->local;
-	loc    = &local->loc;
+        prev   = cookie;
+        subvol = prev->this;
+        conf   = this->private;
+        local  = frame->local;
+        loc    = &local->loc;
 
         if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"lookup of %s on %s (following linkfile) failed (%s)",
-			local->loc.path, subvol->name, strerror (op_errno));
+                gf_log (this->name, GF_LOG_INFO,
+                        "lookup of %s on %s (following linkfile) failed (%s)",
+                        local->loc.path, subvol->name, strerror (op_errno));
 
-		dht_lookup_everywhere (frame, this, loc);
-		return 0;
-	}
+                /* If cached subvol returned ENOTCONN, do not do
+                lookup_everywhere. We need to make sure linkfile does not get
+                removed, which can take away the namespace, and subvol is
+                anyways down. */
 
-        /* TODO: assert type is non-dir and non-linkfile */
+                if (op_errno != ENOTCONN)
+                        goto err;
+                else
+                        goto unwind;
+        }
 
-	if (stbuf->st_nlink == 1)
-		stbuf->st_mode |= S_ISVTX;
-        dht_itransform (this, prev->this, stbuf->st_ino, &stbuf->st_ino);
+        if (check_is_dir (inode, stbuf, xattr)) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "lookup of %s on %s (following linkfile) reached dir",
+                        local->loc.path, subvol->name);
+                goto err;
+        }
 
-	layout = dht_layout_for_subvol (this, prev->this);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no pre-set layout for subvolume %s",
-			prev->this->name);
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
+        if (check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "lookup of %s on %s (following linkfile) reached link",
+                        local->loc.path, subvol->name);
+                goto err;
+        }
+
+        if (uuid_compare (local->gfid, stbuf->ia_gfid)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: gfid different on data file on %s",
+                        local->loc.path, subvol->name);
+                goto err;
+        }
+
+        if ((stbuf->ia_nlink == 1)
+            && (conf && conf->unhashed_sticky_bit)) {
+                stbuf->ia_prot.sticky = 1;
+        }
 
-	inode_ctx_put (inode, this, (uint64_t)(long)layout);
+        ret = dht_layout_preset (this, prev->this, inode);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "failed to set layout for subvolume %s",
+                        prev->this->name);
+                op_ret   = -1;
+                op_errno = EINVAL;
+        }
+
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+
+unwind:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                          postparent);
+
+        return 0;
 
+err:
+        dht_lookup_everywhere (frame, this, loc);
 out:
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, inode, stbuf, xattr);
+        return 0;
+}
+
+
+int
+dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+        int           call_cnt = 0;
+        int           i = 0;
+        dht_conf_t   *conf = NULL;
+        dht_local_t  *local = NULL;
+        int           ret = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", loc, unwind);
+
+        conf = this->private;
+        local = frame->local;
+
+        call_cnt        = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
+
+        local->layout = dht_layout_new (this, conf->subvolume_cnt);
+        if (!local->layout) {
+                goto unwind;
+        }
+
+        if (local->xattr != NULL) {
+                dict_unref (local->xattr);
+                local->xattr = NULL;
+        }
 
+        if (!uuid_is_null (local->gfid)) {
+                ret = dict_set_static_bin (local->xattr_req, "gfid-req",
+                                           local->gfid, 16);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set gfid", local->loc.path);
+        }
+
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_lookup_dir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->lookup,
+                            &local->loc, local->xattr_req);
+        }
+        return 0;
+unwind:
+        DHT_STACK_UNWIND (lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+out:
         return 0;
+
 }
 
 
 int
 dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno,
-                inode_t *inode, struct stat *stbuf, dict_t *xattr)
+                inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                struct iatt *postparent)
 {
-	dht_layout_t *layout      = NULL;
-        char          is_linkfile = 0;
-        char          is_dir      = 0;
-        xlator_t     *subvol      = NULL;
-        dht_conf_t   *conf        = NULL;
-        dht_local_t  *local       = NULL;
-        loc_t        *loc         = NULL;
-        int           i           = 0;
-        call_frame_t *prev        = NULL;
-	int           call_cnt    = 0;
+        char          is_linkfile   = 0;
+        char          is_dir        = 0;
+        xlator_t     *subvol        = NULL;
+        dht_conf_t   *conf          = NULL;
+        dht_local_t  *local         = NULL;
+        loc_t        *loc           = NULL;
+        call_frame_t *prev          = NULL;
+        int           ret           = 0;
+        dht_layout_t *parent_layout = NULL;
 
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
 
         conf  = this->private;
 
@@ -512,93 +1261,140 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local = frame->local;
         loc   = &local->loc;
 
-	if (ENTRY_MISSING (op_ret, op_errno)) {
-		if (conf->search_unhashed) {
-			local->op_errno = ENOENT;
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
-		}
-	}
-
- 	if (op_ret == 0) {
- 		is_dir      = check_is_dir (inode, stbuf, xattr);
- 		if (is_dir) {
- 			local->inode = inode_ref (inode);
- 			local->xattr = dict_ref (xattr);
- 		}
- 	}
-
- 	if (is_dir || (op_ret == -1 && op_errno == ENOTCONN)) {
-		call_cnt        = conf->subvolume_cnt;
- 		local->call_cnt = call_cnt;
-		
- 		local->layout = dht_layout_new (this, conf->subvolume_cnt);
- 		if (!local->layout) {
- 			op_ret   = -1;
- 			op_errno = ENOMEM;
- 			gf_log (this->name, GF_LOG_ERROR,
- 				"memory allocation failed :(");
- 			goto out;
- 		}
-		
-		for (i = 0; i < call_cnt; i++) {
-			STACK_WIND (frame, dht_lookup_dir_cbk,
-				    conf->subvolumes[i],
-				    conf->subvolumes[i]->fops->lookup,
-				    &local->loc, local->xattr_req);
-		}
- 		return 0;
- 	}
- 
-        if (op_ret == -1)
-                goto out;
+        /* This is required for handling stale linkfile deletion,
+         * or any more call which happens from this 'loc'.
+         */
+        if (!op_ret && uuid_is_null (local->gfid))
+                memcpy (local->gfid, stbuf->ia_gfid, 16);
+
+        if (ENTRY_MISSING (op_ret, op_errno)) {
+                gf_log (this->name, GF_LOG_TRACE, "Entry %s missing on subvol"
+                        " %s", loc->path, prev->this->name);
+                if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_ON) {
+                        local->op_errno = ENOENT;
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
+                }
+                if ((conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) &&
+                    (loc->parent)) {
+                        ret = dht_inode_ctx_layout_get (loc->parent, this,
+                                                        &parent_layout);
+                        if (ret || !parent_layout)
+                                goto out;
+                        if (parent_layout->search_unhashed) {
+                                local->op_errno = ENOENT;
+                                dht_lookup_everywhere (frame, this, loc);
+                                return 0;
+                        }
+                }
+        }
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
+        if (op_ret == 0) {
+                is_dir      = check_is_dir (inode, stbuf, xattr);
+                if (is_dir) {
+                        local->inode = inode_ref (inode);
+                        local->xattr = dict_ref (xattr);
+                }
+        }
 
-        if (!is_dir && !is_linkfile) {
-                /* non-directory and not a linkfile */
+        if (is_dir || (op_ret == -1 && op_errno == ENOTCONN)) {
+                dht_lookup_directory (frame, this, &local->loc);
+                return 0;
+        }
 
-		dht_itransform (this, prev->this, stbuf->st_ino,
-				&stbuf->st_ino);
-
-		layout = dht_layout_for_subvol (this, prev->this);
-		if (!layout) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"no pre-set layout for subvolume %s",
-				prev->this->name);
-			op_ret   = -1;
-			op_errno = EINVAL;
-			goto out;
-		}
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "Lookup of %s for subvolume"
+                       " %s failed with error %s", loc->path, prev->this->name,
+                       strerror (op_errno));
+                goto out;
+        }
 
-                inode_ctx_put (inode, this, (uint64_t)(long)layout);
-                goto out; 
-	}
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
 
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+        if (!is_linkfile) {
+                /* non-directory and not a linkfile */
 
-                if (!subvol) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "linkfile not having link subvolume. path=%s",
-                                loc->path);
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
+                ret = dht_layout_preset (this, prev->this, inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "could not set pre-set layout for subvolume %s",
+                                prev->this->name);
+                        op_ret   = -1;
+                        op_errno = EINVAL;
+                        goto out;
                 }
+                goto out;
+        }
 
-		STACK_WIND (frame, dht_lookup_linkfile_cbk,
-			    subvol, subvol->fops->lookup,
-			    &local->loc, local->xattr_req);
+        subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "linkfile not having link subvolume. path=%s",
+                        loc->path);
+                dht_lookup_everywhere (frame, this, loc);
+                return 0;
         }
 
+        STACK_WIND (frame, dht_lookup_linkfile_cbk,
+                    subvol, subvol->fops->lookup,
+                    &local->loc, local->xattr_req);
+
         return 0;
 
 out:
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, inode, stbuf, xattr);
+        /*
+         * FIXME: postparent->ia_size and postparent->st_blocks do not have
+         * correct values. since, postparent corresponds to a directory these
+         * two members should have values equal to sum of corresponding values
+         * from each of the subvolume. See dht_iatt_merge for reference.
+         */
+
+        if (!op_ret && local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                          postparent);
+err:
         return 0;
 }
 
+/* For directories, check if acl xattrs have been requested (by the acl xlator),
+ * if not, request for them. These xattrs are needed for dht dir self-heal to
+ * perform proper self-healing of dirs
+ */
+void
+dht_check_and_set_acl_xattr_req (inode_t *inode, dict_t *xattr_req)
+{
+        int     ret = 0;
+
+        GF_ASSERT (inode);
+        GF_ASSERT (xattr_req);
+
+        if (inode->ia_type != IA_IFDIR)
+                return;
+
+        if (!dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_ACCESS_XATTR);
+        }
+
+        if (!dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_DEFAULT_XATTR);
+        }
+
+        return;
+}
 
 int
 dht_lookup (call_frame_t *frame, xlator_t *this,
@@ -606,125 +1402,163 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
 {
         xlator_t     *subvol = NULL;
         xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
         dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_conf_t   *conf = NULL;
         int           ret    = -1;
         int           op_errno = -1;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-	int           call_cnt = 0;
-
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
+        loc_t         new_loc = {0,};
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
-	conf = this->private;
-
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        conf = this->private;
+        if (!conf)
+                goto err;
 
-        ret = loc_dup (loc, &local->loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "copying location failed for path=%s",
-                        loc->path);
+        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
+        if (!local) {
+                op_errno = ENOMEM;
                 goto err;
         }
-	
-	if (xattr_req) {
-		local->xattr_req = dict_ref (xattr_req);
-	} else {
-		local->xattr_req = dict_new ();
-	}
 
-	hashed_subvol = dht_subvol_get_hashed (this, loc);
-	cached_subvol = dht_subvol_get_cached (this, loc->inode);
+        ret = dht_filter_loc_subvol_key (this, loc, &new_loc,
+                                         &hashed_subvol);
+        if (ret) {
+                loc_wipe (&local->loc);
+                ret = loc_dup (&new_loc, &local->loc);
 
-	local->cached_subvol = cached_subvol;
-	local->hashed_subvol = hashed_subvol;
+                /* we no more need 'new_loc' entries */
+                loc_wipe (&new_loc);
 
-        if (is_revalidate (loc)) {
-		layout = dht_layout_get (this, loc->inode);
+                /* check if loc_dup() is successful */
+                if (ret == -1) {
+                        op_errno = errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "copying location failed for path=%s",
+                                loc->path);
+                        goto err;
+                }
+        }
 
+        if (xattr_req) {
+                local->xattr_req = dict_ref (xattr_req);
+        } else {
+                local->xattr_req = dict_new ();
+        }
+
+        if (uuid_is_null (loc->pargfid) && !uuid_is_null (loc->gfid) &&
+            !__is_root_gfid (loc->inode->gfid)) {
+                local->cached_subvol = NULL;
+                dht_discover (frame, this, loc);
+                return 0;
+        }
+
+        if (!hashed_subvol)
+                hashed_subvol = dht_subvol_get_hashed (this, loc);
+        local->hashed_subvol = hashed_subvol;
+
+        if (is_revalidate (loc)) {
+                layout = local->layout;
                 if (!layout) {
-                        gf_log (this->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "revalidate without cache. path=%s",
                                 loc->path);
                         op_errno = EINVAL;
                         goto err;
                 }
 
-		if (layout->gen && (layout->gen < conf->gen)) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"incomplete layout failure for path=%s",
-				loc->path);
-			op_errno = EAGAIN;
-			goto err;
-		}
+                if (layout->gen && (layout->gen < conf->gen)) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "incomplete layout failure for path=%s",
+                                loc->path);
+
+                        dht_layout_unref (this, local->layout);
+                        local->layout = NULL;
+                        local->cached_subvol = NULL;
+                        goto do_fresh_lookup;
+                }
+
+                local->inode = inode_ref (loc->inode);
+
+                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
+                 *       revalidates directly go to the cached-subvolume.
+                 */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
+
+                if (IA_ISDIR (local->inode->ia_type)) {
+                        local->call_cnt = call_cnt = conf->subvolume_cnt;
+                        for (i = 0; i < call_cnt; i++) {
+                                STACK_WIND (frame, dht_revalidate_cbk,
+                                            conf->subvolumes[i],
+                                            conf->subvolumes[i]->fops->lookup,
+                                            loc, local->xattr_req);
+                        }
+                        return 0;
+                }
+
+                call_cnt = local->call_cnt = layout->cnt;
 
-		local->inode    = inode_ref (loc->inode);
-		local->st_ino   = loc->inode->ino;
-		
-		local->call_cnt = layout->cnt;
-		call_cnt = local->call_cnt;
-		
-		/* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
-		 *       revalidates directly go to the cached-subvolume.
-		 */
-		ret = dict_set_uint32 (local->xattr_req, 
-				       "trusted.glusterfs.dht", 4 * 4);
-
-		for (i = 0; i < layout->cnt; i++) {
+                /* need it for self-healing linkfiles which is
+                   'in-migration' state */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       GLUSTERFS_OPEN_FD_COUNT, 4);
+
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+
+		for (i = 0; i < call_cnt; i++) {
 			subvol = layout->list[i].xlator;
-			
+
 			STACK_WIND (frame, dht_revalidate_cbk,
 				    subvol, subvol->fops->lookup,
-				    loc, local->xattr_req);
+				    &local->loc, local->xattr_req);
 
-			if (!--call_cnt)
-				break;
 		}
         } else {
-		/* TODO: remove the hard-coding */
-		ret = dict_set_uint32 (local->xattr_req, 
-				       "trusted.glusterfs.dht", 4 * 4);
+        do_fresh_lookup:
+                /* TODO: remove the hard-coding */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
+
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->link_xattr_name, 256);
+
+                /* need it for self-healing linkfiles which is
+                   'in-migration' state */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       GLUSTERFS_OPEN_FD_COUNT, 4);
 
-		ret = dict_set_uint32 (local->xattr_req, 
-				       "trusted.glusterfs.dht.linkto", 256);
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
 
                 if (!hashed_subvol) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"no subvolume in layout for path=%s, "
-				"checking on all the subvols to see if "
-				"it is a directory", loc->path);
- 			call_cnt        = conf->subvolume_cnt;
- 			local->call_cnt = call_cnt;
- 			
- 			local->layout = dht_layout_new (this, conf->subvolume_cnt);
- 			if (!local->layout) {
- 				op_errno = ENOMEM;
- 				gf_log (this->name, GF_LOG_ERROR,
- 					"memory allocation failed :(");
- 				goto err;
- 			}
-
-			for (i = 0; i < call_cnt; i++) {
- 				STACK_WIND (frame, dht_lookup_dir_cbk,
- 					    conf->subvolumes[i],
- 					    conf->subvolumes[i]->fops->lookup,
- 					    &local->loc, local->xattr_req);
- 			}
- 			return 0;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "no subvolume in layout for path=%s, "
+                                "checking on all the subvols to see if "
+                                "it is a directory", loc->path);
+                        call_cnt        = conf->subvolume_cnt;
+                        local->call_cnt = call_cnt;
+
+                        local->layout = dht_layout_new (this,
+                                                        conf->subvolume_cnt);
+                        if (!local->layout) {
+                                op_errno = ENOMEM;
+                                goto err;
+                        }
+
+                        for (i = 0; i < call_cnt; i++) {
+                                STACK_WIND (frame, dht_lookup_dir_cbk,
+                                            conf->subvolumes[i],
+                                            conf->subvolumes[i]->fops->lookup,
+                                            &local->loc, local->xattr_req);
+                        }
+                        return 0;
                 }
 
                 STACK_WIND (frame, dht_lookup_cbk,
@@ -735,2011 +1569,2630 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
 
 int
-dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int op_ret, int op_errno, struct stat *stbuf)
+dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev  = NULL;
 
+        local = frame->local;
+        prev  = cookie;
 
-	local = frame->local;
-	prev = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_ERROR,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_ret   = -1;
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
 
-		dht_stat_merge (this, &local->stbuf, stbuf, prev->this);
-		
-		if (local->inode)
-			local->stbuf.st_ino = local->inode->ino;
-		local->op_ret = 0;
-	}
+                local->op_ret = 0;
+
+                local->postparent = *postparent;
+                local->preparent = *preparent;
+
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  &local->stbuf);
+        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, NULL);
 
         return 0;
 }
 
 
 int
-dht_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc)
+dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
 
+        xlator_t *cached_subvol = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        local = frame->local;
+        prev  = cookie;
 
-	layout = dht_layout_get (this, loc->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no layout for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        LOCK (&frame->lock);
+        {
+                if ((op_ret == -1) && !((op_errno == ENOENT) ||
+                                        (op_errno == ENOTCONN))) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
 
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = layout->cnt;
+        if (local->op_ret == -1)
+                goto err;
 
-	for (i = 0; i < layout->cnt; i++) {
-		subvol = layout->list[i].xlator;
+        cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
+        if (!cached_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s",
+                        local->loc.path);
+                local->op_errno = EINVAL;
+                goto err;
+        }
 
-		STACK_WIND (frame, dht_attr_cbk,
-			    subvol, subvol->fops->stat,
-			    loc);
-	}
+        STACK_WIND (frame, dht_unlink_cbk,
+                    cached_subvol, cached_subvol->fops->unlink,
+                    &local->loc, local->flags, NULL);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
-
-	return 0;
+        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
+                          NULL, NULL, NULL);
+        return 0;
 }
 
-
 int
-dht_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd)
+dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int op_ret, int op_errno, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-	layout = dht_layout_get (this, fd->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no layout for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"local allocation failed :(");
-		goto err;
-	}
+        local = frame->local;
+        prev = cookie;
 
-	local->inode    = inode_ref (fd->inode);
-	local->call_cnt = layout->cnt;;
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
 
-	for (i = 0; i < layout->cnt; i++) {
-		subvol = layout->list[i].xlator;
-		STACK_WIND (frame, dht_attr_cbk,
-			    subvol, subvol->fops->fstat,
-			    fd);
-	}
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
 
-	return 0;
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
+        }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        return 0;
+}
 
-	return 0;
+static void
+fill_layout_info (dht_layout_t *layout, char *buf)
+{
+        int i = 0;
+        char tmp_buf[128] = {0,};
+
+        for (i = 0; i < layout->cnt; i++) {
+                snprintf (tmp_buf, 128, "(%s %u %u)",
+                          layout->list[i].xlator->name,
+                          layout->list[i].start,
+                          layout->list[i].stop);
+                if (i)
+                        strcat (buf, " ");
+                strcat (buf, tmp_buf);
+        }
 }
 
+void
+dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
+                         char *xattr_buf, int32_t alloc_len,
+                         int flag, char *layout_buf)
+{
+        if (flag && local->xattr_val)
+                snprintf (xattr_buf, alloc_len,
+                          "((<"DHT_PATHINFO_HEADER"%s> %s) (%s-layout %s))",
+                          this->name, local->xattr_val, this->name,
+                          layout_buf);
+        else if (local->xattr_val)
+                snprintf (xattr_buf, alloc_len,
+                          "(<"DHT_PATHINFO_HEADER"%s> %s)",
+                          this->name, local->xattr_val);
+        else if (flag)
+                snprintf (xattr_buf, alloc_len, "(%s-layout %s)",
+                          this->name, layout_buf);
+}
 
 int
-dht_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode)
+dht_vgetxattr_alloc_and_fill (dht_local_t *local, dict_t *xattr, xlator_t *this,
+                              int op_errno)
 {
-	dht_layout_t *layout = NULL;
-	dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-	int           i = -1;
+        int      ret       = -1;
+        char    *value     = NULL;
+        int32_t  plen      = 0;
 
+        ret = dict_get_str (xattr, local->xsel, &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Subvolume %s returned -1 (%s)", this->name,
+                        strerror (op_errno));
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                goto out;
+        }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        local->alloc_len += strlen(value);
 
-	layout = dht_layout_get (this, loc->inode);
+        if (!local->xattr_val) {
+                local->alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
+                local->xattr_val = GF_CALLOC (local->alloc_len, sizeof (char),
+                                              gf_common_mt_char);
+                if (!local->xattr_val) {
+                        ret = -1;
+                        goto out;
+                }
+        }
 
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no layout for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        if (local->xattr_val) {
+                plen = strlen (local->xattr_val);
+                if (plen) {
+                        /* extra byte(s) for \0 to be safe */
+                        local->alloc_len += (plen + 2);
+                        local->xattr_val = GF_REALLOC (local->xattr_val,
+                                                       local->alloc_len);
+                        if (!local->xattr_val) {
+                                ret = -1;
+                                goto out;
+                        }
+                }
 
-	if (!layout_is_sane (layout)) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"layout is not sane for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+                (void) strcat (local->xattr_val, value);
+                (void) strcat (local->xattr_val, " ");
+                local->op_ret = 0;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        ret = 0;
 
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = layout->cnt;
+ out:
+        return ret;
+}
 
-	for (i = 0; i < layout->cnt; i++) {
-		STACK_WIND (frame, dht_attr_cbk,
-			    layout->list[i].xlator,
-			    layout->list[i].xlator->fops->chmod,
-			    loc, mode);
-	}
+int
+dht_vgetxattr_fill_and_set (dht_local_t *local, dict_t **dict, xlator_t *this,
+                            gf_boolean_t flag)
+{
+        int   ret             = -1;
+        char *xattr_buf       = NULL;
+        char layout_buf[8192] = {0,};
 
-	return 0;
+        if (flag)
+                fill_layout_info (local->layout, layout_buf);
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        *dict = dict_new ();
+        if (!*dict)
+                goto out;
 
-	return 0;
-}
+        local->xattr_val[strlen (local->xattr_val) - 1] = '\0';
+
+        /* we would need max this many bytes to create xattr string
+         * extra 40 bytes is just an estimated amount of additional
+         * space required as we include translator name and some
+         * spaces, brackets etc. when forming the pathinfo string.
+         *
+         * For node-uuid we just don't have all the pretty formatting,
+         * but since this is a generic routine for pathinfo & node-uuid
+         * we dont have conditional space allocation and try to be
+         * generic
+         */
+        local->alloc_len += (2 * strlen (this->name))
+                + strlen (layout_buf)
+                + 40;
+        xattr_buf = GF_CALLOC (local->alloc_len, sizeof (char),
+                               gf_common_mt_char);
+        if (!xattr_buf)
+                goto out;
+
+        if (XATTR_IS_PATHINFO (local->xsel)) {
+                (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
+                                                local->alloc_len, flag,
+                                                layout_buf);
+        } else if (XATTR_IS_NODE_UUID (local->xsel)) {
+                (void) snprintf (xattr_buf, local->alloc_len, "%s",
+                                 local->xattr_val);
+        } else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unknown local->xsel (%s)", local->xsel);
+                goto out;
+        }
+
+        ret = dict_set_dynstr (*dict, local->xsel, xattr_buf);
+        GF_FREE (local->xattr_val);
 
+ out:
+        return ret;
+}
 
 int
-dht_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid)
+dht_vgetxattr_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-	dht_layout_t *layout = NULL;
-	dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-	int           i = -1;
+        int          ret           = 0;
+        dht_local_t *local         = NULL;
+        int          this_call_cnt = 0;
+        dict_t      *dict          = NULL;
 
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        local = frame->local;
 
-	layout = dht_layout_get (this, loc->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no layout for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        LOCK (&frame->lock);
+        {
+                this_call_cnt = --local->call_cnt;
+                if (op_ret < 0) {
+                        if (op_errno != ENOTCONN) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "getxattr err (%s) for dir",
+                                        strerror (op_errno));
+				local->op_ret = -1;
+				local->op_errno = op_errno;
+                        }
 
-	if (!layout_is_sane (layout)) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"layout is not sane for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+                        goto unlock;
+                }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+                ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                                    op_errno);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "alloc or fill failure");
+        }
+ unlock:
+        UNLOCK (&frame->lock);
 
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = layout->cnt;
+        if (!is_last_call (this_call_cnt))
+                goto out;
 
-	for (i = 0; i < layout->cnt; i++) {
-		STACK_WIND (frame, dht_attr_cbk,
-			    layout->list[i].xlator,
-			    layout->list[i].xlator->fops->chown,
-			    loc, uid, gid);
-	}
+        /* -- last call: do patch ups -- */
 
-	return 0;
+        if (local->op_ret == -1) {
+                goto unwind;
+        }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, _gf_true);
+        if (ret)
+                goto unwind;
 
-	return 0;
-}
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
 
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
+ out:
+        return 0;
+}
 
 int
-dht_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode)
+dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-	dht_layout_t *layout = NULL;
-	dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-	int           i = -1;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
+        dht_local_t  *local         = NULL;
+        int           ret           = 0;
+        dict_t       *dict          = NULL;
+        call_frame_t *prev          = NULL;
+        gf_boolean_t  flag          = _gf_true;
 
-	layout = dht_layout_get (this, fd->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no layout for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local = frame->local;
+        prev = cookie;
 
-	if (!layout_is_sane (layout)) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"layout is not sane for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        if (op_ret < 0) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                gf_log (this->name, GF_LOG_ERROR, "Subvolume %s returned -1 "
+                        "(%s)", prev->this->name, strerror (op_errno));
+                goto unwind;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                            op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "alloc or fill failure");
+                goto unwind;
+        }
 
-	local->inode = inode_ref (fd->inode);
-	local->call_cnt = layout->cnt;
+        flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
 
-	for (i = 0; i < layout->cnt; i++) {
-		STACK_WIND (frame, dht_attr_cbk,
-			    layout->list[i].xlator,
-			    layout->list[i].xlator->fops->fchmod,
-			    fd, mode);
-	}
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, flag);
+        if (ret)
+                goto unwind;
 
-	return 0;
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno,
+                          NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
 
-	return 0;
+        return 0;
 }
 
-
 int
-dht_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid)
+dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
-	dht_layout_t *layout = NULL;
-	dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-	int           i = -1;
+        int   ret   = 0;
+        char *value = NULL;
+
+        if (op_ret != -1) {
+                ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
+                if (!ret) {
+                        ret = dict_set_str (xattr, GF_XATTR_LINKINFO_KEY, value);
+                        if (!ret)
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "failed to set linkinfo");
+                }
+        }
 
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        return 0;
+}
 
-	layout = dht_layout_get (this, fd->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no layout for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+int
+dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        int             this_call_cnt = 0;
+        dht_local_t     *local = NULL;
+        dht_conf_t      *conf = NULL;
 
-	if (!layout_is_sane (layout)) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"layout is not sane for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        conf = this->private;
+        local = frame->local;
 
-	local->inode = inode_ref (fd->inode);
-	local->call_cnt = layout->cnt;
+        this_call_cnt = dht_frame_return (frame);
 
-	for (i = 0; i < layout->cnt; i++) {
-		STACK_WIND (frame, dht_attr_cbk,
-			    layout->list[i].xlator,
-			    layout->list[i].xlator->fops->fchown,
-			    fd, uid, gid);
-	}
+        if (!xattr || (op_ret == -1))
+                goto out;
 
-	return 0;
+        if (dict_get (xattr, conf->xattr_name)) {
+                dict_del (xattr, conf->xattr_name);
+        }
+        local->op_ret = 0;
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        if (!local->xattr) {
+                local->xattr = dict_copy_with_ref (xattr, NULL);
+        } else {
+                /* first aggregate everything into xattr and then copy into
+                 * local->xattr. This is required as we want to have
+                 * 'local->xattr' as the proper final dictionary passed above
+                 * distribute xlator.
+                 */
+                dht_aggregate_xattr (xattr, local->xattr);
+                local->xattr = dict_copy (xattr, local->xattr);
+        }
+out:
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+                                  local->xattr, NULL);
+        }
+        return 0;
+}
 
-	return 0;
+int32_t
+dht_getxattr_unwind (call_frame_t *frame,
+                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+{
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
-dht_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2])
+dht_getxattr_get_real_filename_cbk (call_frame_t *frame, void *cookie,
+				    xlator_t *this, int op_ret, int op_errno,
+				    dict_t *xattr, dict_t *xdata)
 {
-	dht_layout_t *layout = NULL;
-	dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-	int           i = -1;
+        int             this_call_cnt = 0;
+        dht_local_t     *local = NULL;
 
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        local = frame->local;
 
-	layout = dht_layout_get (this, loc->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no layout for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+	if (op_ret != -1) {
+		if (local->xattr)
+			dict_unref (local->xattr);
+		local->xattr = dict_ref (xattr);
 
-	if (!layout_is_sane (layout)) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"layout is not sane for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
+		if (local->xattr_req)
+			dict_unref (local->xattr_req);
+		local->xattr_req = dict_ref (xdata);
 	}
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
+	this_call_cnt = dht_frame_return (frame);
+	if (is_last_call (this_call_cnt)) {
+		DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+				  local->xattr, local->xattr_req);
 	}
 
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = layout->cnt;
+	return 0;
+}
 
-	for (i = 0; i < layout->cnt; i++) {
-		STACK_WIND (frame, dht_attr_cbk,
-			    layout->list[i].xlator,
-			    layout->list[i].xlator->fops->utimens,
-			    loc, tv);
-	}
 
-	return 0;
+int
+dht_getxattr_get_real_filename (call_frame_t *frame, xlator_t *this,
+				loc_t *loc, const char *key, dict_t *xdata)
+{
+	dht_local_t     *local = NULL;
+	int              i = 0;
+	dht_layout_t    *layout = NULL;
+	int              cnt = 0;
+	xlator_t        *subvol = NULL;
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+
+	local = frame->local;
+	layout = local->layout;
+
+	cnt = local->call_cnt = layout->cnt;
+
+	local->op_ret = -1;
+	local->op_errno = ENODATA;
+
+	for (i = 0; i < cnt; i++) {
+		subvol = layout->list[i].xlator;
+		STACK_WIND (frame, dht_getxattr_get_real_filename_cbk,
+			    subvol, subvol->fops->getxattr,
+			    loc, key, xdata);
+	}
 
 	return 0;
 }
 
 
 int
-dht_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset)
+dht_getxattr (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, const char *key, dict_t *xdata)
+#define DHT_IS_DIR(layout)  (layout->cnt > 1)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
 
+        xlator_t     *subvol        = NULL;
+        xlator_t     *hashed_subvol = NULL;
+        xlator_t     *cached_subvol = NULL;
+        dht_conf_t   *conf          = NULL;
+        dht_local_t  *local         = NULL;
+        dht_layout_t *layout        = NULL;
+        xlator_t     **sub_volumes  = NULL;
+        int           op_errno      = -1;
+        int           i             = 0;
+        int           cnt           = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        VALIDATE_OR_GOTO (this->private, err);
 
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = 1;
+        conf   = this->private;
 
-	STACK_WIND (frame, dht_attr_cbk,
-		    subvol, subvol->fops->truncate,
-		    loc, offset);
+        local = dht_local_init (frame, loc, NULL, GF_FOP_GETXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
 
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+                goto err;
+        }
 
-	return 0;
-}
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "layout is NULL");
+                op_errno = ENOENT;
+                goto err;
+        }
 
+        if (key) {
+                local->key = gf_strdup (key);
+                if (!local->key) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+        }
 
-int
-dht_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
+	if (key &&
+	    (strncmp (key, GF_XATTR_GET_REAL_FILENAME_KEY,
+		      strlen (GF_XATTR_GET_REAL_FILENAME_KEY)) == 0)
+	    && DHT_IS_DIR(layout)) {
+		dht_getxattr_get_real_filename (frame, this, loc, key, xdata);
+		return 0;
+	}
 
+        /* for file use cached subvolume (obviously!): see if {}
+         * below
+         * for directory:
+         *  wind to all subvolumes and exclude subvolumes which
+         *  return ENOTCONN (in callback)
+         *
+         * NOTE: Don't trust inode here, as that may not be valid
+         *       (until inode_link() happens)
+         */
+        if (key && DHT_IS_DIR(layout) &&
+            ((strcmp (key, GF_XATTR_PATHINFO_KEY) == 0)
+             || (strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0))) {
+                (void) strncpy (local->xsel, key, 256);
+                cnt = local->call_cnt = layout->cnt;
+                for (i = 0; i < cnt; i++) {
+                        subvol = layout->list[i].xlator;
+                        STACK_WIND (frame, dht_vgetxattr_dir_cbk,
+                                    subvol, subvol->fops->getxattr,
+                                    loc, key, NULL);
+                }
+                return 0;
+        }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        /* node-uuid or pathinfo for files */
+        if (key && ((strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
+                    || (strcmp (key, GF_XATTR_PATHINFO_KEY) == 0))) {
+                cached_subvol = local->cached_subvol;
+                (void) strncpy (local->xsel, key, 256);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+                local->call_cnt = 1;
+                STACK_WIND (frame, dht_vgetxattr_cbk, cached_subvol,
+                            cached_subvol->fops->getxattr, loc, key, NULL);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+                return 0;
+        }
 
-	local->inode = inode_ref (fd->inode);
-	local->call_cnt = 1;
+        if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
+                hashed_subvol = dht_subvol_get_hashed (this, loc);
+                if (!hashed_subvol) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get"
+                                "hashed subvol for %s", loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
 
-	STACK_WIND (frame, dht_attr_cbk,
-		    subvol, subvol->fops->ftruncate,
-		    fd, offset);
+                cached_subvol = dht_subvol_get_cached (this, loc->inode);
+                if (!cached_subvol) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get"
+                                "cached subvol for %s", loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
 
-	return 0;
+                if (hashed_subvol == cached_subvol) {
+                        op_errno = ENODATA;
+                        goto err;
+                }
+                if (hashed_subvol) {
+                        STACK_WIND (frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
+                                    hashed_subvol->fops->getxattr, loc,
+                                    GF_XATTR_PATHINFO_KEY, NULL);
+                        return 0;
+                }
+                op_errno = ENODATA;
+                goto err;
+        }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        if (key && (!strcmp (GF_XATTR_MARKER_KEY, key))
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                if (DHT_IS_DIR(layout)) {
+                        cnt = layout->cnt;
+                } else {
+                        cnt = 1;
+                }
 
-	return 0;
-}
+                sub_volumes = alloca ( cnt * sizeof (xlator_t *));
+                for (i = 0; i < cnt; i++)
+                        *(sub_volumes + i) = layout->list[i].xlator;
 
+                if (cluster_getmarkerattr (frame, this, loc, key,
+                                           local, dht_getxattr_unwind,
+                                           sub_volumes, cnt,
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           conf->vol_uuid)) {
+                        op_errno = EINVAL;
+                        goto err;
+                }
 
-int
-dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	     int op_ret, int op_errno)
-{
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
+                return 0;
+        }
 
+        if (key && *conf->vol_uuid) {
+                if ((match_uuid_local (key, conf->vol_uuid) == 0) &&
+                    (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                        if (DHT_IS_DIR(layout)) {
+                                cnt = layout->cnt;
+                        } else {
+                                cnt = 1;
+                        }
+                        sub_volumes = alloca ( cnt * sizeof (xlator_t *));
+                        for (i = 0; i < cnt; i++)
+                                sub_volumes[i] = layout->list[i].xlator;
+
+                        if (cluster_getmarkerattr (frame, this, loc, key,
+                                                   local, dht_getxattr_unwind,
+                                                   sub_volumes, cnt,
+                                                   MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
+                                                   conf->vol_uuid)) {
+                                op_errno = EINVAL;
+                                goto err;
+                        }
+
+                        return 0;
+                }
+        }
 
-	local = frame->local;
-	prev = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_ERROR,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
+        if (DHT_IS_DIR(layout)) {
+                cnt = local->call_cnt = layout->cnt;
+        } else {
+                cnt = local->call_cnt  = 1;
+        }
 
-		local->op_ret = 0;
-	}
-unlock:
-	UNLOCK (&frame->lock);
+        for (i = 0; i < cnt; i++) {
+                subvol = layout->list[i].xlator;
+                STACK_WIND (frame, dht_getxattr_cbk,
+                            subvol, subvol->fops->getxattr,
+                            loc, key, NULL);
+        }
+        return 0;
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno);
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
-
+#undef DHT_IS_DIR
 
 int
-dht_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask)
+dht_fgetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, const char *key, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
+        xlator_t     *subvol        = NULL;
+        dht_local_t  *local         = NULL;
+        dht_layout_t *layout        = NULL;
+        int           op_errno      = -1;
+        int           i             = 0;
+        int           cnt           = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FGETXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+                goto err;
+        }
 
-	local->call_cnt = 1;
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "layout is NULL");
+                op_errno = ENOENT;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_err_cbk,
-		    subvol, subvol->fops->access,
-		    loc, mask);
+        if (key) {
+                local->key = gf_strdup (key);
+                if (!local->key) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+        }
 
-	return 0;
+        if ((fd->inode->ia_type == IA_IFDIR)
+            && (strncmp (key, GF_XATTR_LOCKINFO_KEY,
+                         strlen (GF_XATTR_LOCKINFO_KEY) != 0))) {
+                cnt = local->call_cnt = layout->cnt;
+        } else {
+                cnt = local->call_cnt  = 1;
+        }
+
+        for (i = 0; i < cnt; i++) {
+                subvol = layout->list[i].xlator;
+                STACK_WIND (frame, dht_getxattr_cbk,
+                            subvol, subvol->fops->fgetxattr,
+                            fd, key, NULL);
+        }
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
-
 int
-dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno, const char *path)
+dht_fsetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, path);
+        xlator_t     *subvol   = NULL;
+        dht_local_t  *local    = NULL;
+        int           op_errno = EINVAL;
+        dht_conf_t   *conf     = NULL;
 
-        return 0;
-}
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
+        conf = this->private;
 
-int
-dht_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
 
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local->call_cnt = 1;
 
-	STACK_WIND (frame, dht_readlink_cbk,
-		    subvol, subvol->fops->readlink,
-		    loc, size);
+        STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr,
+                    fd, xattr, flags, NULL);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
-int
-dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno, dict_t *xattr)
+static int
+dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *xdata)
 {
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, xattr);
+        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
-
 int
-dht_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *key)
+dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
+        int           i     = -1;
+        int           ret   = -1;
+        char         *value = NULL;
+        dht_local_t  *local = NULL;
+        dht_conf_t   *conf  = NULL;
+        call_frame_t *prev  = NULL;
+        int           this_call_cnt = 0;
 
+        local = frame->local;
+        prev = cookie;
+        conf = this->private;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        if (op_ret == -1)
+                goto out;
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
 
-	STACK_WIND (frame, dht_getxattr_cbk,
-		    subvol, subvol->fops->getxattr,
-		    loc, key);
+        ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
+        if (ret)
+                goto out;
 
-	return 0;
+        if (!strcmp (value, local->key)) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (conf->subvolumes[i] == prev->this)
+                                conf->decommissioned_bricks[i] = prev->this;
+                }
+        }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+out:
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP, NULL);
+        }
+        return 0;
 
-	return 0;
 }
 
-
 int
 dht_setxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, dict_t *xattr, int flags)
+              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
+        xlator_t     *subvol   = NULL;
+        dht_local_t  *local    = NULL;
+        dht_conf_t   *conf     = NULL;
+        dht_layout_t *layout   = NULL;
+        int           i        = 0;
+        int           op_errno = EINVAL;
+        int           ret      = -1;
+        data_t       *tmp      = NULL;
+        uint32_t      dir_spread = 0;
+        char          value[4096] = {0,};
+        gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
+        int           call_cnt = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        conf   = this->private;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
 
-	local->call_cnt = 1;
+        local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_err_cbk,
-		    subvol, subvol->fops->setxattr,
-		    loc, xattr, flags);
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	return 0;
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        local->call_cnt = call_cnt = layout->cnt;
 
-	return 0;
-}
+        tmp = dict_get (xattr, "distribute.migrate-data");
+        if (tmp) {
+                if (IA_ISDIR (loc->inode->ia_type)) {
+                        op_errno = ENOTSUP;
+                        goto err;
+                }
 
+                /* TODO: need to interpret the 'value' for more meaning
+                   (ie, 'target' subvolume given there, etc) */
+                memcpy (value, tmp->data, tmp->len);
+                if (strcmp (value, "force") == 0)
+                       forced_rebalance =
+                                GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
 
-int
-dht_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *key)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
+                if (conf->decommission_in_progress)
+                        forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
 
+                local->rebalance.target_node = dht_subvol_get_hashed (this, loc);
+                if (!local->rebalance.target_node) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "hashed subvol for %s", loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+                local->rebalance.from_subvol = local->cached_subvol;
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+                if (local->rebalance.target_node == local->rebalance.from_subvol) {
+                        op_errno = EEXIST;
+                        goto err;
+                }
+                if (local->rebalance.target_node) {
+                        local->flags = forced_rebalance;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+                        ret = dht_start_rebalance_task (this, frame);
+                        if (!ret)
+                                return 0;
 
-	local->call_cnt = 1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to create a new synctask",
+                                loc->path);
+                }
+                op_errno = EINVAL;
+                goto err;
 
-	STACK_WIND (frame, dht_err_cbk,
-		    subvol, subvol->fops->removexattr,
-		    loc, key);
+        }
 
-	return 0;
+        tmp = dict_get (xattr, "decommission-brick");
+        if (tmp) {
+                /* This operation should happen only on '/' */
+                if (!__is_root_gfid (loc->inode->gfid)) {
+                        op_errno = ENOTSUP;
+                        goto err;
+                }
+
+                memcpy (value, tmp->data, ((tmp->len < 4095) ? tmp->len : 4095));
+                local->key = gf_strdup (value);
+                local->call_cnt = conf->subvolume_cnt;
+
+                for (i = 0 ; i < conf->subvolume_cnt; i++) {
+                        /* Get the pathinfo, and then compare */
+                        STACK_WIND (frame, dht_checking_pathinfo_cbk,
+                                    conf->subvolumes[i],
+                                    conf->subvolumes[i]->fops->getxattr,
+                                    loc, GF_XATTR_PATHINFO_KEY, NULL);
+                }
+                return 0;
+        }
+
+        tmp = dict_get (xattr, GF_XATTR_FIX_LAYOUT_KEY);
+        if (tmp) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "fixing the layout of %s", loc->path);
+
+                ret = dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
+                                                layout);
+                if (ret) {
+                        op_errno = ENOTCONN;
+                        goto err;
+                }
+                return ret;
+        }
+
+        tmp = dict_get (xattr, "distribute.directory-spread-count");
+        if (tmp) {
+                /* Setxattr value is packed as 'binary', not string */
+                memcpy (value, tmp->data, ((tmp->len < 4095)?tmp->len:4095));
+                ret = gf_string2uint32 (value, &dir_spread);
+                if (!ret && ((dir_spread <= conf->subvolume_cnt) &&
+                             (dir_spread > 0))) {
+                        layout->spread_cnt = dir_spread;
+
+                        ret = dht_fix_directory_layout (frame,
+                                                        dht_common_setxattr_cbk,
+                                                        layout);
+                        if (ret) {
+                                op_errno = ENOTCONN;
+                                goto err;
+                        }
+                        return ret;
+                }
+                gf_log (this->name, GF_LOG_ERROR,
+                        "wrong 'directory-spread-count' value (%s)", value);
+                op_errno = ENOTSUP;
+                goto err;
+        }
+
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_err_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->setxattr,
+                            loc, xattr, flags, xdata);
+        }
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int op_ret, int op_errno, fd_t *fd)
+dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
 
+        local = frame->local;
+        prev = cookie;
 
-	local = frame->local;
-	prev = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_ERROR,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
 
-		local->op_ret = 0;
-	}
+                local->op_ret = 0;
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  local->fd);
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
+        }
 
         return 0;
 }
 
 
 int
-dht_open (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc, int flags, fd_t *fd)
+dht_removexattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, const char *key, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-	int           ret = -1;
+        xlator_t     *subvol = NULL;
         int           op_errno = -1;
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        int           call_cnt = 0;
+        dht_conf_t   *conf = NULL;
 
+        int i;
 
-        VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        conf = this->private;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
-	local->fd = fd_ref (fd);
-	ret = loc_dup (loc, &local->loc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
 
-	local->call_cnt = 1;
+        local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_fd_cbk,
-		    subvol, subvol->fops->open,
-		    loc, flags, fd);
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	return 0;
+        layout = local->layout;
+        if (!local->layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        local->call_cnt = call_cnt = layout->cnt;
+        local->key = gf_strdup (key);
 
-	return 0;
-}
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_removexattr_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->removexattr,
+                            loc, key, NULL);
+        }
 
+        return 0;
 
-int
-dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	       int op_ret, int op_errno,
-	       struct iovec *vector, int count, struct stat *stbuf)
-{
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf);
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
 
         return 0;
 }
 
-
 int
-dht_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t off)
+dht_fremovexattr (call_frame_t *frame, xlator_t *this,
+                  fd_t *fd, const char *key, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        xlator_t     *subvol = NULL;
         int           op_errno = -1;
+        dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        int           call_cnt = 0;
+        dht_conf_t   *conf = 0;
 
+        int i;
 
-        VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, dht_readv_cbk,
-		    subvol, subvol->fops->readv,
-		    fd, size, off);
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL);
-
-	return 0;
-}
-
+        VALIDATE_OR_GOTO (this->private, err);
 
-int
-dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, struct stat *stbuf)
-{
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, stbuf);
+        conf = this->private;
 
-        return 0;
-}
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
+        VALIDATE_OR_GOTO (frame, err);
 
-int
-dht_writev (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, struct iovec *vector, int count, off_t off)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FREMOVEXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for inode=%s",
+                        uuid_utoa (fd->inode->gfid));
+                op_errno = EINVAL;
+                goto err;
+        }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        layout = local->layout;
+        if (!local->layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for inode=%s", uuid_utoa (fd->inode->gfid));
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local->call_cnt = call_cnt = layout->cnt;
+        local->key = gf_strdup (key);
 
-	STACK_WIND (frame, dht_writev_cbk,
-		    subvol, subvol->fops->writev,
-		    fd, vector, count, off);
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_removexattr_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->fremovexattr,
+                            fd, key, NULL);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, 0);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+            int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
 
+        local = frame->local;
+        prev = cookie;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt))
+                DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
+                                  local->fd, NULL);
 
-	local->fd = fd_ref (fd);
-	local->call_cnt = 1;
+        return 0;
+}
 
-	STACK_WIND (frame, dht_err_cbk,
-		    subvol, subvol->fops->flush, fd);
+/*
+ * dht_normalize_stats -
+ */
+static void
+dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
+                     unsigned long frsize)
+{
+        double factor = 0;
 
-	return 0;
+        if (buf->f_bsize != bsize) {
+                buf->f_bsize = bsize;
+        }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
+        if (buf->f_frsize != frsize) {
+                factor = ((double) buf->f_frsize) / frsize;
+                buf->f_frsize = frsize;
+                buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
+                buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
+                buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
 
-	return 0;
+        }
 }
 
-
 int
-dht_fsync (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, int datasync)
+dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
+        dht_local_t *local         = NULL;
+        int          this_call_cnt = 0;
+        int          bsize         = 0;
+        int          frsize        = 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocatoin failed :(");
-		goto err;
-	}
-	local->call_cnt = 1;
 
-	STACK_WIND (frame, dht_err_cbk,
-		    subvol, subvol->fops->fsync,
-		    fd, datasync);
+        local = frame->local;
 
-	return 0;
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        goto unlock;
+                }
+                local->op_ret = 0;
+
+                if (local->statvfs.f_bsize != 0) {
+                        bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
+                        frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
+                        dht_normalize_stats(&local->statvfs, bsize, frsize);
+                        dht_normalize_stats(statvfs, bsize, frsize);
+                } else {
+                        local->statvfs.f_bsize    = statvfs->f_bsize;
+                        local->statvfs.f_frsize   = statvfs->f_frsize;
+                }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
+                local->statvfs.f_blocks  += statvfs->f_blocks;
+                local->statvfs.f_bfree   += statvfs->f_bfree;
+                local->statvfs.f_bavail  += statvfs->f_bavail;
+                local->statvfs.f_files   += statvfs->f_files;
+                local->statvfs.f_ffree   += statvfs->f_ffree;
+                local->statvfs.f_favail  += statvfs->f_favail;
+                local->statvfs.f_fsid     = statvfs->f_fsid;
+                local->statvfs.f_flag     = statvfs->f_flag;
+                local->statvfs.f_namemax  = statvfs->f_namemax;
 
-	return 0;
-}
+        }
+unlock:
+        UNLOCK (&frame->lock);
 
 
-int
-dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	    int op_ret, int op_errno, struct flock *flock)
-{
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, flock);
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt))
+                DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
+                                  &local->statvfs, xdata);
 
         return 0;
 }
 
 
 int
-dht_lk (call_frame_t *frame, xlator_t *this,
-	fd_t *fd, int cmd, struct flock *flock)
+dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        xlator_t     *subvol = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
-
+        int           i = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        conf = this->private;
 
-	STACK_WIND (frame, dht_lk_cbk,
-		    subvol, subvol->fops->lk,
-		    fd, cmd, flock);
+        local = dht_local_init (frame, NULL, NULL, GF_FOP_STATFS);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	return 0;
+        if (IA_ISDIR (loc->inode->ia_type)) {
+                local->call_cnt = conf->subvolume_cnt;
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        STACK_WIND (frame, dht_statfs_cbk,
+                                    conf->subvolumes[i],
+                                    conf->subvolumes[i]->fops->statfs, loc,
+                                    xdata);
+                }
+                return 0;
+        }
 
-	return 0;
-}
+        subvol = dht_subvol_get_cached (this, loc->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-/* gf_lk no longer exists 
-int
-dht_gf_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	    int op_ret, int op_errno, struct flock *flock)
-{
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, flock);
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_statfs_cbk,
+                    subvol, subvol->fops->statfs, loc, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_gf_lk (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int cmd, struct flock *flock)
+dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+             dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
-
+        int           i = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, dht_gf_lk_cbk,
-		    subvol, subvol->fops->gf_lk,
-		    fd, cmd, flock);
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
-
-	return 0;
-}
-*/
+        conf = this->private;
 
-int
-dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, struct statvfs *statvfs)
-{
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
+        local = dht_local_init (frame, loc, fd, GF_FOP_OPENDIR);
+        if (!local) {
+                op_errno = ENOMEM;
 
+                goto err;
+        }
 
-	local = frame->local;
+        local->call_cnt = conf->subvolume_cnt;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-		local->op_ret = 0;
-
-		/* TODO: normalize sizes */
-		local->statvfs.f_bsize    = statvfs->f_bsize;
-		local->statvfs.f_frsize   = statvfs->f_frsize;
-
-		local->statvfs.f_blocks  += statvfs->f_blocks;
-		local->statvfs.f_bfree   += statvfs->f_bfree;
-		local->statvfs.f_bavail  += statvfs->f_bavail;
-		local->statvfs.f_files   += statvfs->f_files;
-		local->statvfs.f_ffree   += statvfs->f_ffree;
-		local->statvfs.f_favail  += statvfs->f_favail;
-		local->statvfs.f_fsid     = statvfs->f_fsid;
-		local->statvfs.f_flag     = statvfs->f_flag;
-		local->statvfs.f_namemax  = statvfs->f_namemax;
-
-	}
-unlock:
-	UNLOCK (&frame->lock);
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_fd_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->opendir,
+                            loc, fd, xdata);
+        }
 
+        return 0;
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  &local->statvfs);
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
 
 
 int
-dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-	int           i = -1;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	conf = this->private;
+        dht_local_t  *local = NULL;
+        gf_dirent_t   entries;
+        gf_dirent_t  *orig_entry = NULL;
+        gf_dirent_t  *entry = NULL;
+        call_frame_t *prev = NULL;
+        xlator_t     *next_subvol = NULL;
+        off_t         next_offset = 0;
+        int           count = 0;
+        dht_layout_t *layout = 0;
+        dht_conf_t   *conf   = NULL;
+        xlator_t     *subvol = 0;
+        int           ret    = 0;
+
+        INIT_LIST_HEAD (&entries.list);
+        prev = cookie;
+        local = frame->local;
+        conf  = this->private;
 
-	local = dht_local_init (frame);
-	local->call_cnt = conf->subvolume_cnt;
+        if (op_ret < 0)
+                goto done;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_statfs_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->statfs, loc);
-	}
+        if (!local->layout)
+                local->layout = dht_layout_get (this, local->fd->inode);
 
-	return 0;
+        layout = local->layout;
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
+                next_offset = orig_entry->d_off;
+                if (check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
+                    (prev->this != local->first_up_subvol)) {
+                        continue;
+                }
+                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
+                                       orig_entry->dict,
+                                       conf->link_xattr_name)) {
+                        continue;
+                }
 
-	return 0;
-}
+                entry = gf_dirent_for_name (orig_entry->d_name);
+                if (!entry) {
 
+                        goto unwind;
+                }
 
-int
-dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
-{
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
-	int           ret = -1;
-        int           op_errno = -1;
-	int           i = -1;
+                /* Do this if conf->search_unhashed is set to "auto" */
+                if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
+                        subvol = dht_layout_search (this, layout,
+                                                    orig_entry->d_name);
+                        if (!subvol || (subvol != prev->this)) {
+                                /* TODO: Count the number of entries which need
+                                   linkfile to prove its existence in fs */
+                                layout->search_unhashed++;
+                        }
+                }
 
+                dht_itransform (this, prev->this, orig_entry->d_off,
+                                &entry->d_off);
+
+                entry->d_stat = orig_entry->d_stat;
+                entry->d_ino  = orig_entry->d_ino;
+                entry->d_type = orig_entry->d_type;
+                entry->d_len  = orig_entry->d_len;
+
+                if (orig_entry->dict)
+                        entry->dict = dict_ref (orig_entry->dict);
+
+                /* making sure we set the inode ctx right with layout,
+                   currently possible only for non-directories, so for
+                   directories don't set entry inodes */
+                if (!IA_ISDIR(entry->d_stat.ia_type)) {
+                        ret = dht_layout_preset (this, prev->this,
+                                                 orig_entry->inode);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to link the layout in inode");
+                        entry->inode = inode_ref (orig_entry->inode);
+                } else if (orig_entry->inode) {
+                        dht_inode_ctx_time_update (orig_entry->inode, this,
+                                                   &entry->d_stat, 1);
+                }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+                list_add_tail (&entry->list, &entries.list);
+                count++;
+        }
+        op_ret = count;
+        /* We need to ensure that only the last subvolume's end-of-directory
+         * notification is respected so that directory reading does not stop
+         * before all subvolumes have been read. That could happen because the
+         * posix for each subvolume sends a ENOENT on end-of-directory but in
+         * distribute we're not concerned only with a posix's view of the
+         * directory but the aggregated namespace' view of the directory.
+         */
+        if (prev->this != dht_last_up_subvol (this))
+                op_errno = 0;
 
-	conf = this->private;
+done:
+        if (count == 0) {
+                /* non-zero next_offset means that
+                   EOF is not yet hit on the current subvol
+                */
+                if (next_offset == 0) {
+                        next_subvol = dht_subvol_next (this, prev->this);
+                } else {
+                        next_subvol = prev->this;
+                }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+                if (!next_subvol) {
+                        goto unwind;
+                }
 
-	local->fd = fd_ref (fd);
-	ret = loc_dup (loc, &local->loc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+		if (conf->readdir_optimize == _gf_true) {
+                        if (next_subvol != local->first_up_subvol) {
+                                ret = dict_set_int32 (local->xattr,
+                                                      GF_READDIR_SKIP_DIRS, 1);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+					         "dict set failed");
+		        } else {
+                                 dict_del (local->xattr,
+                                           GF_READDIR_SKIP_DIRS);
+                        }
+                }
 
-	local->call_cnt = conf->subvolume_cnt;
+                STACK_WIND (frame, dht_readdirp_cbk,
+                            next_subvol, next_subvol->fops->readdirp,
+                            local->fd, local->size, next_offset,
+                            local->xattr);
+                return 0;
+        }
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_fd_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->opendir,
-			    loc, fd);
-	}
+unwind:
+        if (op_ret < 0)
+                op_ret = 0;
 
-	return 0;
+        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        gf_dirent_free (&entries);
 
-	return 0;
+        return 0;
 }
 
 
+
 int
 dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, gf_dirent_t *orig_entries)
+                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
+                 dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	gf_dirent_t   entries;
-	gf_dirent_t  *orig_entry = NULL;
-	gf_dirent_t  *entry = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *subvol = NULL;
-	xlator_t     *next = NULL;
-	dht_layout_t *layout = NULL;
-	int           count = 0;
+        dht_local_t  *local = NULL;
+        gf_dirent_t   entries;
+        gf_dirent_t  *orig_entry = NULL;
+        gf_dirent_t  *entry = NULL;
+        call_frame_t *prev = NULL;
+        xlator_t     *next_subvol = NULL;
+        off_t         next_offset = 0;
+        int           count = 0;
+        dht_layout_t *layout = 0;
+        xlator_t     *subvol = 0;
 
+        INIT_LIST_HEAD (&entries.list);
+        prev = cookie;
+        local = frame->local;
 
-	INIT_LIST_HEAD (&entries.list);
-	prev = cookie;
-	local = frame->local;
+        if (op_ret < 0)
+                goto done;
 
-	if (op_ret < 0)
-		goto done;
+        if (!local->layout)
+                local->layout = dht_layout_get (this, local->fd->inode);
 
-	layout = dht_layout_get (this, local->fd->inode);
+        layout = local->layout;
 
-	list_for_each_entry (orig_entry, &orig_entries->list, list) {
-		subvol = dht_layout_search (this, layout, orig_entry->d_name);
+        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
+                next_offset = orig_entry->d_off;
 
-		if (!subvol || subvol == prev->this) {
-			entry = gf_dirent_for_name (orig_entry->d_name);
-			if (!entry) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"memory allocation failed :(");
-				goto unwind;
-			}
+                subvol = dht_layout_search (this, layout, orig_entry->d_name);
 
-			dht_itransform (this, subvol, orig_entry->d_ino,
-					&entry->d_ino);
-			dht_itransform (this, subvol, orig_entry->d_off,
-					&entry->d_off);
+                if (!subvol || (subvol == prev->this)) {
+                        entry = gf_dirent_for_name (orig_entry->d_name);
+                        if (!entry) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "memory allocation failed :(");
+                                goto unwind;
+                        }
 
-			entry->d_type = orig_entry->d_type;
-			entry->d_len  = orig_entry->d_len;
+                        dht_itransform (this, prev->this, orig_entry->d_off,
+                                        &entry->d_off);
 
-			list_add_tail (&entry->list, &entries.list);
-			count++;
-		}
-	}
-	op_ret = count;
+                        entry->d_ino  = orig_entry->d_ino;
+                        entry->d_type = orig_entry->d_type;
+                        entry->d_len  = orig_entry->d_len;
+
+                        list_add_tail (&entry->list, &entries.list);
+                        count++;
+                }
+        }
+        op_ret = count;
+        /* We need to ensure that only the last subvolume's end-of-directory
+         * notification is respected so that directory reading does not stop
+         * before all subvolumes have been read. That could happen because the
+         * posix for each subvolume sends a ENOENT on end-of-directory but in
+         * distribute we're not concerned only with a posix's view of the
+         * directory but the aggregated namespace' view of the directory.
+         */
+        if (prev->this != dht_last_up_subvol (this))
+                op_errno = 0;
 
 done:
-	if (count == 0) {
-		next = dht_subvol_next (this, prev->this);
-		if (!next) {
-			goto unwind;
-		}
+        if (count == 0) {
+                /* non-zero next_offset means that
+                   EOF is not yet hit on the current subvol
+                */
+                if (next_offset == 0) {
+                        next_subvol = dht_subvol_next (this, prev->this);
+                } else {
+                        next_subvol = prev->this;
+                }
 
-		STACK_WIND (frame, dht_readdir_cbk,
-			    next, next->fops->readdir,
-			    local->fd, local->size, 0);
-		return 0;
-	}
+                if (!next_subvol) {
+                        goto unwind;
+                }
+
+                STACK_WIND (frame, dht_readdir_cbk,
+                            next_subvol, next_subvol->fops->readdir,
+                            local->fd, local->size, next_offset, NULL);
+                return 0;
+        }
 
 unwind:
-	if (op_ret < 0)
-		op_ret = 0;
+        if (op_ret < 0)
+                op_ret = 0;
 
-	DHT_STACK_UNWIND (frame, op_ret, op_errno, &entries);
+        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
 
-	gf_dirent_free (&entries);
+        gf_dirent_free (&entries);
 
         return 0;
 }
 
 
 int
-dht_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t yoff)
+dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t yoff, int whichop, dict_t *dict)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_local_t  *local  = NULL;
         int           op_errno = -1;
-	xlator_t     *xvol = NULL;
-	off_t         xoff = 0;
-
+        xlator_t     *xvol = NULL;
+        off_t         xoff = 0;
+        int           ret = 0;
+        dht_conf_t   *conf = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	conf = this->private;
-
-	local = dht_local_init (frame);
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        conf = this->private;
 
-	local->fd = fd_ref (fd);
-	local->size = size;
+        local = dht_local_init (frame, NULL, NULL, whichop);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	dht_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
+        local->fd = fd_ref (fd);
+        local->size = size;
+        local->xattr_req = (dict)? dict_ref (dict) : NULL;
+        local->first_up_subvol = dht_first_up_subvol (this);
+
+        dht_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
+
+        /* TODO: do proper readdir */
+        if (whichop == GF_FOP_READDIRP) {
+                if (dict)
+                        local->xattr = dict_ref (dict);
+                else
+                        local->xattr = dict_new ();
+
+                if (local->xattr) {
+                        ret = dict_set_uint32 (local->xattr,
+                                               conf->link_xattr_name, 256);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to set '%s' key",
+                                        conf->link_xattr_name);
+			if (conf->readdir_optimize == _gf_true) {
+                                if (xvol != local->first_up_subvol) {
+				        ret = dict_set_int32 (local->xattr,
+			                               GF_READDIR_SKIP_DIRS, 1);
+				        if (ret)
+					        gf_log (this->name,
+                                                        GF_LOG_ERROR,
+						        "Dict set failed");
+                                } else {
+                                        dict_del (local->xattr,
+                                                  GF_READDIR_SKIP_DIRS);
+                                }
+			}
+                }
 
-	/* TODO: do proper readdir */
-	STACK_WIND (frame, dht_readdir_cbk,
-		    xvol, xvol->fops->readdir,
-		    fd, size, xoff);
+                STACK_WIND (frame, dht_readdirp_cbk, xvol, xvol->fops->readdirp,
+                            fd, size, xoff, local->xattr);
+        } else {
+                STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
+                            fd, size, xoff, local->xattr);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
 
-	return 0;
+        return 0;
+}
+
+
+int
+dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t yoff, dict_t *xdata)
+{
+        int          op = GF_FOP_READDIR;
+        dht_conf_t  *conf = NULL;
+        int          i = 0;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->subvolume_status[i]) {
+                        op = GF_FOP_READDIRP;
+                        break;
+                }
+        }
+
+        if (conf->use_readdirp)
+                op = GF_FOP_READDIRP;
+
+out:
+        dht_do_readdir (frame, this, fd, size, yoff, op, 0);
+        return 0;
 }
 
+int
+dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t yoff, dict_t *dict)
+{
+        dht_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
+        return 0;
+}
+
+
 
 int
 dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno)
+                  int op_ret, int op_errno, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
 
 
-	local = frame->local;
+        local = frame->local;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1)
-			local->op_errno = op_errno;
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1)
+                        local->op_errno = op_errno;
 
-		if (op_ret == 0)
-			local->op_ret = 0;
-	}
-	UNLOCK (&frame->lock);
+                if (op_ret == 0)
+                        local->op_ret = 0;
+        }
+        UNLOCK (&frame->lock);
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno);
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt))
+                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret,
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
 int
-dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              int datasync, dict_t *xdata)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
-	int           i = -1;
-
+        int           i = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	conf = this->private;
+        conf = this->private;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        local = dht_local_init (frame, NULL, NULL, GF_FOP_FSYNCDIR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local->fd = fd_ref (fd);
-	local->call_cnt = conf->subvolume_cnt;
+        local->fd = fd_ref (fd);
+        local->call_cnt = conf->subvolume_cnt;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_fsyncdir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->fsyncdir,
-			    fd, datasync);
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_fsyncdir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->fsyncdir,
+                            fd, datasync, xdata);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno,
-		 inode_t *inode, struct stat *stbuf)
+                 int op_ret, int op_errno,
+                 inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-	call_frame_t *prev = NULL;
-	dht_layout_t *layout = NULL;
-	int           ret = -1;
+        xlator_t     *prev = NULL;
+        int           ret = -1;
+        dht_local_t  *local = NULL;
 
 
-	if (op_ret == -1)
-		goto out;
+        if (op_ret == -1)
+                goto out;
 
-	prev = cookie;
+        local = frame->local;
+        if (!local) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
 
-	dht_itransform (this, prev->this, stbuf->st_ino, &stbuf->st_ino);
-	layout = dht_layout_for_subvol (this, prev->this);
+        prev = cookie;
 
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no pre-set layout for subvolume %s",
-			prev->this->name);
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
+        if (local->loc.parent) {
 
-	ret = inode_ctx_put (inode, this, (uint64_t)(long)layout);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"could not set inode context");
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
+        ret = dht_layout_preset (this, prev, inode);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "could not set pre-set layout for subvolume %s",
+                        prev? prev->name: NULL);
+                op_ret   = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
 out:
-	DHT_STACK_UNWIND (frame, op_ret, op_errno, inode, stbuf);
-	return 0;
+        /*
+         * FIXME: ia_size and st_blocks of preparent and postparent do not have
+         * correct values. since, preparent and postparent buffers correspond
+         * to a directory these two members should have values equal to sum of
+         * corresponding values from each of the subvolume.
+         * See dht_iatt_merge for reference.
+         */
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf,
+                          preparent, postparent, xdata);
+        return 0;
 }
 
+int
+dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this,
+                               int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *stbuf,
+                               struct iatt *preparent, struct iatt *postparent,
+                               dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        xlator_t     *cached_subvol = NULL;
+
+        if (op_ret == -1)
+                goto err;
+
+        local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        cached_subvol = local->cached_subvol;
+
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)cached_subvol,
+                           cached_subvol, cached_subvol->fops->mknod,
+                           &local->loc, local->mode, local->rdev, local->umask,
+                           local->params);
+
+        return 0;
+err:
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL);
+        return 0;
+}
 
 int
 dht_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t rdev)
+           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
 {
-	xlator_t  *subvol = NULL;
-	int        op_errno = -1;
+        xlator_t    *subvol = NULL;
+        int          op_errno = -1;
+        xlator_t    *avail_subvol = NULL;
+        dht_local_t *local = NULL;
 
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        dht_get_du_info (frame, this, loc);
 
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"creating %s on %s", loc->path, subvol->name);
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_newfile_cbk,
-		    subvol, subvol->fops->mknod,
-		    loc, mode, rdev);
+        if (!dht_is_subvol_filled (this, subvol)) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "creating %s on %s", loc->path, subvol->name);
 
-	return 0;
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol,
+                                   subvol, subvol->fops->mknod, loc, mode,
+                                   rdev, umask, params);
+        } else {
+
+                avail_subvol = dht_free_disk_available_subvol (this, subvol,
+                                                               local);
+                if (avail_subvol != subvol) {
+                        /* Choose the minimum filled volume, and create the
+                           files there */
+
+                        local->params = dict_ref (params);
+                        local->cached_subvol = avail_subvol;
+                        local->mode = mode;
+                        local->rdev = rdev;
+                        local->umask = umask;
+                        dht_linkfile_create (frame,
+                                             dht_mknod_linkfile_create_cbk,
+                                             this, avail_subvol, subvol, loc);
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "creating %s on %s", loc->path, subvol->name);
+
+                        STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                                           (void *)subvol, subvol,
+                                           subvol->fops->mknod, loc, mode,
+                                           rdev, umask, params);
+                }
+        }
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkname, loc_t *loc)
+             const char *linkname, loc_t *loc, mode_t umask, dict_t *params)
 {
-	xlator_t  *subvol = NULL;
-	int        op_errno = -1;
+        xlator_t    *subvol = NULL;
+        int          op_errno = -1;
+        dht_local_t *local = NULL;
 
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        local = dht_local_init (frame, loc, NULL, GF_FOP_SYMLINK);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"creating %s on %s", loc->path, subvol->name);
+        gf_log (this->name, GF_LOG_TRACE,
+                "creating %s on %s", loc->path, subvol->name);
 
-	STACK_WIND (frame, dht_newfile_cbk,
-		    subvol, subvol->fops->symlink,
-		    linkname, loc);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->symlink, linkname, loc, umask,
+                           params);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (link, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+            dict_t *xdata)
 {
-	xlator_t    *cached_subvol = NULL;
-	xlator_t    *hashed_subvol = NULL;
-	int          op_errno = -1;
-	dht_local_t *local = NULL;
-
-
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        xlator_t    *cached_subvol = NULL;
+        xlator_t    *hashed_subvol = NULL;
+        int          op_errno = -1;
+        dht_local_t *local = NULL;
 
-	cached_subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!cached_subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	hashed_subvol = dht_subvol_get_hashed (this, loc);
-	if (!hashed_subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        if (dht_filter_loc_subvol_key (this, loc, &local->loc,
+                                       &cached_subvol)) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "unlinking %s on %s (given path %s)",
+                        local->loc.path, cached_subvol->name, loc->path);
+                STACK_WIND (frame, dht_unlink_cbk,
+                            cached_subvol, cached_subvol->fops->unlink,
+                            &local->loc, xflag, xdata);
+                goto done;
+        }
 
-	local->call_cnt = 1;
-	if (hashed_subvol != cached_subvol)
-		local->call_cnt++;
+        local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK);
+        if (!local) {
+                op_errno = ENOMEM;
 
-	STACK_WIND (frame, dht_err_cbk,
-		    cached_subvol, cached_subvol->fops->unlink, loc);
+                goto err;
+        }
 
-	if (hashed_subvol != cached_subvol)
-		STACK_WIND (frame, dht_err_cbk,
-			    hashed_subvol, hashed_subvol->fops->unlink, loc);
+        hashed_subvol = dht_subvol_get_hashed (this, loc);
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	return 0;
+        cached_subvol = local->cached_subvol;
+        if (!cached_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
+        local->flags = xflag;
+        if (hashed_subvol != cached_subvol) {
+                STACK_WIND (frame, dht_unlink_linkfile_cbk,
+                            hashed_subvol, hashed_subvol->fops->unlink, loc,
+                            xflag, xdata);
+        } else {
+                STACK_WIND (frame, dht_unlink_cbk,
+                            cached_subvol, cached_subvol->fops->unlink, loc,
+                            xflag, xdata);
+        }
+done:
+        return 0;
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int op_ret, int op_errno,
-	      inode_t *inode, struct stat *stbuf)
+              int op_ret, int op_errno,
+              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+              struct iatt *postparent, dict_t *xdata)
 {
         call_frame_t *prev = NULL;
-	dht_layout_t *layout = NULL;
-	dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        dht_local_t  *local = NULL;
 
         prev = cookie;
-	local = frame->local;
+
+        local = frame->local;
 
         if (op_ret == -1)
                 goto out;
 
-	layout = dht_layout_for_subvol (this, prev->this);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no pre-set layout for subvolume %s",
-			prev->this->name);
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
-
-	stbuf->st_ino = local->loc.inode->ino;
+        layout = dht_layout_for_subvol (this, prev->this);
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no pre-set layout for subvolume %s",
+                        prev->this->name);
+                op_ret   = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
 
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, inode, stbuf);
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                          postparent, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno,
-		       inode_t *inode, struct stat *stbuf)
+                       int op_ret, int op_errno,
+                       inode_t *inode, struct iatt *stbuf,
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	xlator_t     *srcvol = NULL;
+        dht_local_t  *local = NULL;
+        xlator_t     *srcvol = NULL;
 
+        if (op_ret == -1)
+                goto err;
 
-	if (op_ret == -1)
-		goto err;
-
-	local = frame->local;
-	srcvol = local->linkfile.srcvol;
+        local = frame->local;
+        srcvol = local->linkfile.srcvol;
 
-	STACK_WIND (frame, dht_link_cbk,
-		    srcvol, srcvol->fops->link,
-		    &local->loc, &local->loc2);
+        STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
+                    &local->loc, &local->loc2, xdata);
 
-	return 0;
+        return 0;
 
 err:
-	DHT_STACK_UNWIND (frame, op_ret, op_errno, inode, stbuf);
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                          postparent, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc)
-{
-	xlator_t    *cached_subvol = NULL;
-	xlator_t    *hashed_subvol = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
-	dht_local_t *local = NULL;
-
-
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (oldloc, err);
-	VALIDATE_OR_GOTO (newloc, err);
-
-	cached_subvol = dht_subvol_get_cached (this, oldloc->inode);
-	if (!cached_subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", oldloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        xlator_t    *cached_subvol = NULL;
+        xlator_t    *hashed_subvol = NULL;
+        int          op_errno = -1;
+        int          ret = -1;
+        dht_local_t *local = NULL;
 
-	hashed_subvol = dht_subvol_get_hashed (this, newloc);
-	if (!hashed_subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			newloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (oldloc, err);
+        VALIDATE_OR_GOTO (newloc, err);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        local = dht_local_init (frame, oldloc, NULL, GF_FOP_LINK);
+        if (!local) {
+                op_errno = ENOMEM;
 
-	ret = loc_copy (&local->loc, oldloc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+                goto err;
+        }
 
-	ret = loc_copy (&local->loc2, newloc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        cached_subvol = local->cached_subvol;
+        if (!cached_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", oldloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	if (hashed_subvol != cached_subvol) {
-		dht_linkfile_create (frame, dht_link_linkfile_cbk,
-				     cached_subvol, hashed_subvol, newloc);
-	} else {
-		STACK_WIND (frame, dht_link_cbk,
-			    cached_subvol, cached_subvol->fops->link,
-			    oldloc, newloc);
-	}
+        hashed_subvol = dht_subvol_get_hashed (this, newloc);
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        newloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	return 0;
+        ret = loc_copy (&local->loc2, newloc);
+        if (ret == -1) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        if (hashed_subvol != cached_subvol) {
+                uuid_copy (local->gfid, oldloc->inode->gfid);
+                dht_linkfile_create (frame, dht_link_linkfile_cbk, this,
+                                     cached_subvol, hashed_subvol, newloc);
+        } else {
+                STACK_WIND (frame, dht_link_cbk,
+                            cached_subvol, cached_subvol->fops->link,
+                            oldloc, newloc, xdata);
+        }
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno,
-		 fd_t *fd, inode_t *inode, struct stat *stbuf)
+                int op_ret, int op_errno,
+                fd_t *fd, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-	call_frame_t *prev = NULL;
-	dht_layout_t *layout = NULL;
-	int           ret = -1;
-
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        dht_local_t  *local = NULL;
 
-	if (op_ret == -1)
-		goto out;
+        if (op_ret == -1)
+                goto out;
 
-	prev = cookie;
+        local = frame->local;
+        if (!local) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
 
-	dht_itransform (this, prev->this, stbuf->st_ino, &stbuf->st_ino);
-	layout = dht_layout_for_subvol (this, prev->this);
+        prev = cookie;
 
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no pre-set layout for subvolume %s",
-			prev->this->name);
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
 
-	ret = inode_ctx_put (inode, this, (uint64_t)(long)layout);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"could not set inode context");
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
+        ret = dht_layout_preset (this, prev->this, inode);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "could not set preset layout for subvol %s",
+                        prev->this->name);
+                op_ret   = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
-	DHT_STACK_UNWIND (frame, op_ret, op_errno, fd, inode, stbuf);
-	return 0;
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf, preparent,
+                          postparent, NULL);
+        return 0;
 }
 
 
 int
+dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
+                                xlator_t *this,
+                                int32_t op_ret, int32_t op_errno,
+                                inode_t *inode, struct iatt *stbuf,
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        xlator_t     *cached_subvol = NULL;
+
+        if (op_ret == -1)
+                goto err;
+
+        local = frame->local;
+        cached_subvol = local->cached_subvol;
+
+        STACK_WIND (frame, dht_create_cbk,
+                    cached_subvol, cached_subvol->fops->create,
+                    &local->loc, local->flags, local->mode,
+                    local->umask, local->fd, local->params);
+
+        return 0;
+err:
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
+        return 0;
+}
+
+int
 dht_create (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t flags, mode_t mode, fd_t *fd)
+            loc_t *loc, int32_t flags, mode_t mode,
+            mode_t umask, fd_t *fd, dict_t *params)
 {
-	xlator_t  *subvol = NULL;
-	int        op_errno = -1;
+        int          op_errno = -1;
+        xlator_t    *subvol = NULL;
+        dht_local_t *local = NULL;
+        xlator_t    *avail_subvol = NULL;
 
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        dht_get_du_info (frame, this, loc);
 
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"creating %s on %s", loc->path, subvol->name);
+        if (dht_filter_loc_subvol_key (this, loc, &local->loc,
+                                       &subvol)) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "creating %s on %s (got create on %s)",
+                        local->loc.path, subvol->name, loc->path);
+                STACK_WIND (frame, dht_create_cbk,
+                            subvol, subvol->fops->create,
+                            &local->loc, flags, mode, umask, fd, params);
+                goto done;
+        }
 
-	STACK_WIND (frame, dht_create_cbk,
-		    subvol, subvol->fops->create,
-		    loc, flags, mode, fd);
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
 
-	return 0;
+        if (!dht_is_subvol_filled (this, subvol)) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "creating %s on %s", loc->path, subvol->name);
+                STACK_WIND (frame, dht_create_cbk,
+                            subvol, subvol->fops->create,
+                            loc, flags, mode, umask, fd, params);
+                goto done;
+        }
+        /* Choose the minimum filled volume, and create the
+           files there */
+        avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
+        if (avail_subvol != subvol) {
+                local->params = dict_ref (params);
+                local->flags = flags;
+                local->mode = mode;
+                local->umask = umask;
+                local->cached_subvol = avail_subvol;
+                local->hashed_subvol = subvol;
+                gf_log (this->name, GF_LOG_TRACE,
+                        "creating %s on %s (link at %s)", loc->path,
+                        avail_subvol->name, subvol->name);
+                dht_linkfile_create (frame, dht_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
+                goto done;
+        }
+        gf_log (this->name, GF_LOG_TRACE,
+                "creating %s on %s", loc->path, subvol->name);
+        STACK_WIND (frame, dht_create_cbk,
+                    subvol, subvol->fops->create,
+                    loc, flags, mode, umask, fd, params);
+done:
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
+                        xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	dht_local_t   *local = NULL;
-	dht_layout_t  *layout = NULL;
+        dht_local_t   *local = NULL;
+        dht_layout_t  *layout = NULL;
 
+        local = frame->local;
+        layout = local->selfheal.layout;
 
-	local = frame->local;
-	layout = local->selfheal.layout;
+        if (op_ret == 0) {
+                dht_layout_set (this, local->inode, layout);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
 
-	if (op_ret == 0) {
-		inode_ctx_put (local->inode, this, (uint64_t)(long)layout);
-		local->selfheal.layout = NULL;
-		local->stbuf.st_ino = local->st_ino;
-	}
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+        }
 
-	DHT_STACK_UNWIND (frame, op_ret, op_errno,
-			  local->inode, &local->stbuf);
+        DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
+                          local->inode, &local->stbuf, &local->preparent,
+                          &local->postparent, NULL);
 
-	return 0;
+        return 0;
 }
 
 int
 dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	       int op_ret, int op_errno, inode_t *inode, struct stat *stbuf)
+               int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	int           ret = -1;
-	call_frame_t *prev = NULL;
-	dht_layout_t *layout = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        int           ret = -1;
+        gf_boolean_t subvol_filled = _gf_false;
+        call_frame_t *prev = NULL;
+        dht_layout_t *layout = NULL;
 
-	local = frame->local;
-	prev  = cookie;
-	layout = local->layout;
+        local = frame->local;
+        prev  = cookie;
+        layout = local->layout;
 
-	LOCK (&frame->lock);
-	{
-		ret = dht_layout_merge (this, layout, prev->this,
-					op_ret, op_errno, NULL);
+        subvol_filled = dht_is_subvol_filled (this, prev->this);
 
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-		dht_stat_merge (this, &local->stbuf, stbuf, prev->this);
-	}
+        LOCK (&frame->lock);
+        {
+                if (subvol_filled && (op_ret != -1)) {
+                        ret = dht_layout_merge (this, layout, prev->this,
+                                                -1, ENOSPC, NULL);
+                } else {
+			if (op_ret == -1 && op_errno == EEXIST)
+				/* Very likely just a race between mkdir and
+				   self-heal (from lookup of a concurrent mkdir
+				   attempt).
+				   Ignore error for now. layout setting will
+				   anyways fail if this was a different (old)
+				   pre-existing different directory.
+				*/
+				op_ret = 0;
+                        ret = dht_layout_merge (this, layout, prev->this,
+                                                op_ret, op_errno, NULL);
+                }
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to merge layouts", local->loc.path);
+
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        goto unlock;
+                }
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		local->layout = NULL;
-		dht_selfheal_new_directory (frame, dht_mkdir_selfheal_cbk,
-					    layout);
-	}
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                dht_selfheal_new_directory (frame, dht_mkdir_selfheal_cbk,
+                                            layout);
+        }
 
         return 0;
 }
 
 int
-dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie, 
-		      xlator_t *this, int op_ret, int op_errno, 
-		      inode_t *inode, struct stat *stbuf)
+dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int op_ret, int op_errno,
+                      inode_t *inode, struct iatt *stbuf,
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           ret = -1;
-	call_frame_t *prev = NULL;
-	dht_layout_t *layout = NULL;
-	dht_conf_t   *conf = NULL;
-	int           i = 0;
-	xlator_t     *hashed_subvol = NULL;
+        dht_local_t  *local = NULL;
+        int           ret = -1;
+        call_frame_t *prev = NULL;
+        dht_layout_t *layout = NULL;
+        dht_conf_t   *conf = NULL;
+        int           i = 0;
+        xlator_t     *hashed_subvol = NULL;
 
-	local = frame->local;
-	prev  = cookie;
-	layout = local->layout;
-	conf = this->private;
-	hashed_subvol = local->hashed_subvol;
+        VALIDATE_OR_GOTO (this->private, err);
 
-	ret = dht_layout_merge (this, layout, prev->this,
-				op_ret, op_errno, NULL);
+        local = frame->local;
+        prev  = cookie;
+        layout = local->layout;
+        conf = this->private;
+        hashed_subvol = local->hashed_subvol;
+
+        if (uuid_is_null (local->loc.gfid) && !op_ret)
+                uuid_copy (local->loc.gfid, stbuf->ia_gfid);
+
+        if (dht_is_subvol_filled (this, hashed_subvol))
+                ret = dht_layout_merge (this, layout, prev->this,
+                                        -1, ENOSPC, NULL);
+        else
+                ret = dht_layout_merge (this, layout, prev->this,
+                                        op_ret, op_errno, NULL);
+
+        /* TODO: we may have to return from the function
+           if layout merge fails. For now, lets just log an error */
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to merge layouts", local->loc.path);
 
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		goto err;
-	}
-	local->op_ret = 0;
+        if (op_ret == -1) {
+                local->op_errno = op_errno;
+                goto err;
+        }
+        local->op_ret = 0;
 
-	dht_stat_merge (this, &local->stbuf, stbuf, prev->this);
+        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        dht_iatt_merge (this, &local->preparent, preparent, prev->this);
+        dht_iatt_merge (this, &local->postparent, postparent, prev->this);
 
-	local->st_ino = local->stbuf.st_ino;
+        local->call_cnt = conf->subvolume_cnt - 1;
 
-	local->call_cnt = conf->subvolume_cnt - 1;
-	
-	if (local->call_cnt == 0) {
-		local->layout = NULL;
-		dht_selfheal_directory (frame, dht_mkdir_selfheal_cbk,
-					&local->loc, layout);
-	}
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (conf->subvolumes[i] == hashed_subvol)
-			continue;
-		STACK_WIND (frame, dht_mkdir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->mkdir,
-			    &local->loc, local->mode);
-	}
-	return 0;
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, stbuf->ia_gfid);
+        if (local->call_cnt == 0) {
+                dht_selfheal_directory (frame, dht_mkdir_selfheal_cbk,
+                                        &local->loc, layout);
+        }
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == hashed_subvol)
+                        continue;
+                STACK_WIND (frame, dht_mkdir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->mkdir, &local->loc,
+                            local->mode, local->umask, local->params);
+        }
+        return 0;
 err:
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
         return 0;
 }
 
-int
+
+ int
 dht_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode)
+           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
-	int           ret = -1;
-	xlator_t     *hashed_subvol = NULL;
+        xlator_t     *hashed_subvol = NULL;
 
 
         VALIDATE_OR_GOTO (frame, err);
@@ -2747,723 +4200,1013 @@ dht_mkdir (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
         VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	conf = this->private;
+        conf = this->private;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	hashed_subvol = dht_subvol_get_hashed (this, loc);
+        dht_get_du_info (frame, this, loc);
 
-	if (hashed_subvol == NULL) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"hashed subvol not found");
-		op_errno = EINVAL;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, NULL, GF_FOP_MKDIR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local->hashed_subvol = hashed_subvol;
-	local->inode = inode_ref (loc->inode);
-	ret = loc_copy (&local->loc, loc);
-	local->mode = mode;
+        hashed_subvol = dht_subvol_get_hashed (this, loc);
+        if (hashed_subvol == NULL) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "hashed subvol not found for %s",
+                        loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        local->hashed_subvol = hashed_subvol;
+        local->mode = mode;
+        local->umask = umask;
+        local->params = dict_ref (params);
+        local->inode  = inode_ref (loc->inode);
 
-	local->layout = dht_layout_new (this, conf->subvolume_cnt);
-	if (!local->layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        local->layout = dht_layout_new (this, conf->subvolume_cnt);
+        if (!local->layout) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_mkdir_hashed_cbk,
-		    hashed_subvol,
-		    hashed_subvol->fops->mkdir,
-		    loc, mode);
+        STACK_WIND (frame, dht_mkdir_hashed_cbk,
+                    hashed_subvol,
+                    hashed_subvol->fops->mkdir,
+                    loc, mode, umask, params);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int op_ret, int op_errno)
+                        int op_ret, int op_errno, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
 
-	local = frame->local;
-	local->layout = NULL;
+        local = frame->local;
 
-	DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno);
+        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	       int op_ret, int op_errno)
+dht_rmdir_hashed_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-	uint64_t      tmp_layout = 0;
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
-	dht_layout_t *layout = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
 
-	local = frame->local;
-	prev  = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			local->op_ret   = -1;
-
-			if (op_errno != ENOENT)
-				local->need_selfheal = 1;
-
-			gf_log (this->name, GF_LOG_ERROR,
-				"rmdir on %s for %s failed (%s)",
-				prev->this->name, local->loc.path,
-				strerror (op_errno));
-			goto unlock;
-		}
-	}
-unlock:
-	UNLOCK (&frame->lock);
+        local = frame->local;
+        prev  = cookie;
 
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret   = -1;
+                        if (op_errno != ENOENT && op_errno != EACCES) {
+                                local->need_selfheal = 1;
+                        }
+
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "rmdir on %s for %s failed (%s)",
+                                prev->this->name, local->loc.path,
+                                strerror (op_errno));
+                        goto unlock;
+                }
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		if (local->need_selfheal) {
-			inode_ctx_get (local->loc.inode, this, 
-				       &tmp_layout);
-			layout = (dht_layout_t *)(long)tmp_layout;
-
-			/* TODO: neater interface needed below */
-			local->stbuf.st_mode = local->loc.inode->st_mode;
-
-			dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
-					      &local->loc, layout);
-		} else {
-			DHT_STACK_UNWIND (frame, local->op_ret,
-					  local->op_errno);
-		}
-	}
+                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
+
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+               if (local->need_selfheal) {
+                        local->layout =
+                                dht_layout_get (this, local->loc.inode);
+
+                        /* TODO: neater interface needed below */
+                        local->stbuf.ia_type = local->loc.inode->ia_type;
+
+                        uuid_copy (local->gfid, local->loc.inode->gfid);
+                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
+                                              &local->loc, local->layout);
+               } else {
+
+                        if (local->loc.parent) {
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+                        }
+
+                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
+                                          local->op_errno, &local->preparent,
+                                          &local->postparent, NULL);
+               }
+        }
 
         return 0;
 }
 
 
 int
-dht_rmdir_do (call_frame_t *frame, xlator_t *this)
+dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	dht_conf_t   *conf = NULL;
-	int           i = 0;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+        int           done = 0;
 
-	conf = this->private;
-	local = frame->local;
+        local = frame->local;
+        prev  = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret   = -1;
+
+                        if (op_errno != ENOENT && op_errno != EACCES) {
+                                local->need_selfheal = 1;
+                        }
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "rmdir on %s for %s failed (%s)",
+                                prev->this->name, local->loc.path,
+                                strerror (op_errno));
+                        goto unlock;
+                }
 
-	if (local->op_ret == -1)
-		goto err;
+                /* Track if rmdir succeeded on atleast one subvol*/
+                local->fop_succeeded = 1;
+                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
+        }
+unlock:
+        UNLOCK (&frame->lock);
 
-	local->call_cnt = conf->subvolume_cnt;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_rmdir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->rmdir,
-			    &local->loc);
-	}
+        this_call_cnt = dht_frame_return (frame);
 
-	return 0;
+        /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
+        if (local->hashed_subvol && (this_call_cnt == 1)) {
+                done = 1;
+        } else if (!local->hashed_subvol && !this_call_cnt) {
+                done = 1;
+        }
 
-err:
-	DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	return 0;
-}
 
+        if (done) {
+                if (local->need_selfheal && local->fop_succeeded) {
+                        local->layout =
+                                dht_layout_get (this, local->loc.inode);
 
-int
-dht_rmdir_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, gf_dirent_t *entries)
-{
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = -1;
-	call_frame_t *prev = NULL;
+                        /* TODO: neater interface needed below */
+                        local->stbuf.ia_type = local->loc.inode->ia_type;
 
-	local = frame->local;
-	prev  = cookie;
-
-	if (op_ret > 2) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"readdir on %s for %s returned %d entries",
-			prev->this->name, local->loc.path, op_ret);
-		local->op_ret = -1;
-		local->op_errno = ENOTEMPTY;
-	}
+                        uuid_copy (local->gfid, local->loc.inode->gfid);
+                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
+                                              &local->loc, local->layout);
+                } else if (this_call_cnt) {
+                        /* If non-hashed subvol's have responded, proceed */
 
-	this_call_cnt = dht_frame_return (frame);
+                        local->need_selfheal = 0;
+                        STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                                    local->hashed_subvol,
+                                    local->hashed_subvol->fops->rmdir,
+                                    &local->loc, local->flags, NULL);
+                } else if (!this_call_cnt) {
+                        /* All subvol's have responded, proceed */
 
-	if (is_last_call (this_call_cnt)) {
-		dht_rmdir_do (frame, this);
-	}
+                        if (local->loc.parent) {
 
-	return 0;
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+
+                        }
+
+                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
+                                          local->op_errno, &local->preparent,
+                                          &local->postparent, NULL);
+                }
+        }
+
+        return 0;
 }
 
 
 int
-dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, fd_t *fd)
+dht_rmdir_do (call_frame_t *frame, xlator_t *this)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = -1;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        dht_conf_t   *conf = NULL;
+        int           i = 0;
+        xlator_t     *hashed_subvol = NULL;
 
+        VALIDATE_OR_GOTO (this->private, err);
 
-	local = frame->local;
-	prev  = cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"opendir on %s for %s failed (%s)",
-			prev->this->name, local->loc.path,
-			strerror (op_errno));
-		goto err;
-	}
+        conf = this->private;
+        local = frame->local;
 
-	STACK_WIND (frame, dht_rmdir_readdir_cbk,
-		    prev->this, prev->this->fops->readdir,
-		    local->fd, 4096, 0);
+        if (local->op_ret == -1)
+                goto err;
 
-	return 0;
+        local->call_cnt = conf->subvolume_cnt;
 
-err:
-	this_call_cnt = dht_frame_return (frame);
+        /* first remove from non-hashed_subvol */
+        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
 
-	if (is_last_call (this_call_cnt)) {
-		dht_rmdir_do (frame, this);
-	}
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to get hashed "
+                        "subvol for %s",local->loc.path);
+        } else {
+                local->hashed_subvol = hashed_subvol;
+        }
 
-	return 0;
+        /* When DHT has only 1 child */
+        if (conf->subvolume_cnt == 1) {
+                STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                            conf->subvolumes[0],
+                            conf->subvolumes[0]->fops->rmdir,
+                            &local->loc, local->flags, NULL);
+                return 0;
+        }
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (hashed_subvol &&
+                    (hashed_subvol == conf->subvolumes[i]))
+                        continue;
+
+                STACK_WIND (frame, dht_rmdir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->rmdir,
+                            &local->loc, local->flags, NULL);
+        }
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, NULL);
+        return 0;
 }
 
 
 int
-dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_rmdir_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                               int op_ret, int op_errno, struct iatt *preparent,
+                               struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-	int           i = -1;
-	int           ret = -1;
+        dht_local_t    *local = NULL;
+        call_frame_t   *prev = NULL;
+        xlator_t       *src = NULL;
+        call_frame_t   *main_frame = NULL;
+        dht_local_t    *main_local = NULL;
+        int             this_call_cnt = 0;
+
+        local  = frame->local;
+        prev   = cookie;
+        src    = prev->this;
 
+        main_frame = local->main_frame;
+        main_local = main_frame->local;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        if (op_ret == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "unlinked linkfile %s on %s",
+                        local->loc.path, src->name);
+        } else {
+                main_local->op_ret   = -1;
+                main_local->op_errno = op_errno;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "unlink of %s on %s failed (%s)",
+                        local->loc.path, src->name, strerror (op_errno));
+        }
 
-	conf = this->private;
+        this_call_cnt = dht_frame_return (main_frame);
+        if (is_last_call (this_call_cnt))
+                dht_rmdir_do (main_frame, this);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        DHT_STACK_DESTROY (frame);
+        return 0;
+}
 
-	local->call_cnt = conf->subvolume_cnt;
-	local->op_ret   = 0;
 
-	ret = loc_copy (&local->loc, loc);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
+int
+dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, dict_t *xattr, struct iatt *parent)
+{
+        dht_local_t    *local = NULL;
+        call_frame_t   *prev = NULL;
+        xlator_t       *src = NULL;
+        call_frame_t   *main_frame = NULL;
+        dht_local_t    *main_local = NULL;
+        int             this_call_cnt = 0;
+        dht_conf_t     *conf = this->private;
 
-	local->fd = fd_create (local->loc.inode, frame->root->pid);
-	if (!local->fd) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        local = frame->local;
+        prev  = cookie;
+        src   = prev->this;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_rmdir_opendir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->opendir,
-			    loc, local->fd);
-	}
+        main_frame = local->main_frame;
+        main_local = main_frame->local;
 
-	return 0;
+        if (op_ret != 0)
+                goto err;
+
+        if (!check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
+                main_local->op_ret  = -1;
+                main_local->op_errno = ENOTEMPTY;
 
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s on %s found to be not a linkfile (type=0%o)",
+                        local->loc.path, src->name, stbuf->ia_type);
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_rmdir_linkfile_unlink_cbk,
+                    src, src->fops->unlink, &local->loc, 0, NULL);
+        return 0;
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
 
-	return 0;
+        this_call_cnt = dht_frame_return (main_frame);
+        if (is_last_call (this_call_cnt))
+                dht_rmdir_do (main_frame, this);
+
+        DHT_STACK_DESTROY (frame);
+        return 0;
 }
 
 
-static int32_t
-dht_xattrop_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 dict_t *dict)
+int
+dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
+                           gf_dirent_t *entries, xlator_t *src)
 {
-	DHT_STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
+        int                 ret = 0;
+        int                 build_ret = 0;
+        gf_dirent_t        *trav = NULL;
+        call_frame_t       *lookup_frame = NULL;
+        dht_local_t        *lookup_local = NULL;
+        dht_local_t        *local = NULL;
+        dict_t             *xattrs = NULL;
+        dht_conf_t         *conf = this->private;
 
-int32_t
-dht_xattrop (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     gf_xattrop_flags_t flags,
-	     dict_t *dict)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
+        local = frame->local;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        list_for_each_entry (trav, &entries->list, list) {
+                if (strcmp (trav->d_name, ".") == 0)
+                        continue;
+                if (strcmp (trav->d_name, "..") == 0)
+                        continue;
+                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict,
+                                              conf->link_xattr_name)) {
+                        ret++;
+                        continue;
+                }
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+                /* this entry is either a directory which is neither "." nor "..",
+                   or a non directory which is not a linkfile. the directory is to
+                   be treated as non-empty
+                */
+                return 0;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        xattrs = dict_new ();
+        if (!xattrs) {
+                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+                return -1;
+        }
 
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = 1;
+        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
+                        " in dict");
+                if (xattrs)
+                        dict_unref (xattrs);
+                return -1;
+        }
 
-	STACK_WIND (frame,
-		    dht_xattrop_cbk,
-		    subvol, subvol->fops->xattrop,
-		    loc, flags, dict);
+        list_for_each_entry (trav, &entries->list, list) {
+                if (strcmp (trav->d_name, ".") == 0)
+                        continue;
+                if (strcmp (trav->d_name, "..") == 0)
+                        continue;
 
-	return 0;
+                lookup_frame = NULL;
+                lookup_local = NULL;
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+                lookup_frame = copy_frame (frame);
+                if (!lookup_frame) {
+                        /* out of memory, let the rmdir fail
+                           (as non-empty, unfortunately) */
+                        goto err;
+                }
 
-	return 0;
-}
+                lookup_local = mem_get0 (this->local_pool);
+                if (!lookup_local) {
+                        goto err;
+                }
 
-static int32_t
-dht_fxattrop_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  dict_t *dict)
-{
-	DHT_STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
+                lookup_frame->local = lookup_local;
+                lookup_local->main_frame = frame;
 
-int32_t
-dht_fxattrop (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      gf_xattrop_flags_t flags,
-	      dict_t *dict)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
+                build_ret = dht_build_child_loc (this, &lookup_local->loc,
+                                                 &local->loc, trav->d_name);
+                if (build_ret != 0)
+                        goto err;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+                uuid_copy (lookup_local->loc.gfid, trav->d_stat.ia_gfid);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+                gf_log (this->name, GF_LOG_TRACE,
+                        "looking up %s on %s",
+                        lookup_local->loc.path, src->name);
 
-	STACK_WIND (frame,
-		    dht_fxattrop_cbk,
-		    subvol, subvol->fops->fxattrop,
-		    fd, flags, dict);
+                LOCK (&frame->lock);
+                {
+                        local->call_cnt++;
+                }
+                UNLOCK (&frame->lock);
 
-	return 0;
+                STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
+                            src, src->fops->lookup,
+                            &lookup_local->loc, xattrs);
+                ret++;
+        }
+
+        if (xattrs)
+                dict_unref (xattrs);
 
+        return ret;
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
+        if (xattrs)
+                dict_unref (xattrs);
 
-	return 0;
+        DHT_STACK_DESTROY (lookup_frame);
+        return 0;
 }
 
 
-static int32_t
-dht_inodelk_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+int
+dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
-	DHT_STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = -1;
+        call_frame_t *prev = NULL;
+        xlator_t     *src = NULL;
+        int           ret = 0;
+
+        local = frame->local;
+        prev  = cookie;
+        src   = prev->this;
+
+        if (op_ret > 2) {
+                ret = dht_rmdir_is_subvol_empty (frame, this, entries, src);
+
+                switch (ret) {
+                case 0: /* non linkfiles exist */
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "readdir on %s for %s returned %d entries",
+                                prev->this->name, local->loc.path, op_ret);
+                        local->op_ret = -1;
+                        local->op_errno = ENOTEMPTY;
+                        break;
+                default:
+                        /* @ret number of linkfiles are getting unlinked */
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "readdir on %s for %s found %d linkfiles",
+                                prev->this->name, local->loc.path, ret);
+                        break;
+                }
+        }
+
+        this_call_cnt = dht_frame_return (frame);
+
+        if (is_last_call (this_call_cnt)) {
+                dht_rmdir_do (frame, this);
+        }
+
+        return 0;
 }
 
 
-int32_t
-dht_inodelk (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, int32_t cmd, struct flock *lock)
+int
+dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = -1;
+        call_frame_t *prev = NULL;
+        dict_t       *dict = NULL;
+        int           ret = 0;
+        dht_conf_t   *conf = this->private;
 
+        local = frame->local;
+        prev  = cookie;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "opendir on %s for %s failed (%s)",
+                        prev->this->name, local->loc.path,
+                        strerror (op_errno));
+                if (op_errno != ENOENT) {
+                        local->op_ret = -1;
+                        local->op_errno = op_errno;
+                }
+                goto err;
+        }
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        dict = dict_new ();
+        if (!dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto err;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        ret = dict_set_uint32 (dict, conf->link_xattr_name, 256);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set '%s' key",
+                        local->loc.path, conf->link_xattr_name);
 
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = 1;
+        STACK_WIND (frame, dht_rmdir_readdirp_cbk,
+                    prev->this, prev->this->fops->readdirp,
+                    local->fd, 4096, 0, dict);
 
-	STACK_WIND (frame,
-		    dht_inodelk_cbk,
-		    subvol, subvol->fops->inodelk,
-		    loc, cmd, lock);
+        if (dict)
+                dict_unref (dict);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
-
-	return 0;
-}
-
+        this_call_cnt = dht_frame_return (frame);
 
-static int32_t
-dht_finodelk_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno)
+        if (is_last_call (this_call_cnt)) {
+                dht_rmdir_do (frame, this);
+        }
 
-{
-	DHT_STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+        return 0;
 }
 
 
-int32_t
-dht_finodelk (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, int32_t cmd, struct flock *lock)
+int
+dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
+        int           i = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        conf = this->private;
 
+        local = dht_local_init (frame, loc, NULL, GF_FOP_RMDIR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	STACK_WIND (frame,
-		    dht_finodelk_cbk,
-		    subvol, subvol->fops->finodelk,
-		    fd, cmd, lock);
+        local->call_cnt = conf->subvolume_cnt;
+        local->op_ret   = 0;
+        local->fop_succeeded = 0;
 
-	return 0;
+        local->flags = flags;
+
+        local->fd = fd_create (local->loc.inode, frame->root->pid);
+        if (!local->fd) {
+
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_rmdir_opendir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->opendir,
+                            loc, local->fd, NULL);
+        }
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
+                          NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
-
-static int32_t
+int
 dht_entrylk_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-	DHT_STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
+
+int
 dht_entrylk (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, const char *basename,
-	     entrylk_cmd cmd, entrylk_type type)
+             const char *volume, loc_t *loc, const char *basename,
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        xlator_t     *subvol = NULL;
         int           op_errno = -1;
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = 1;
+        local->call_cnt = 1;
 
-	STACK_WIND (frame, dht_entrylk_cbk,
-		    subvol, subvol->fops->entrylk,
-		    loc, basename, cmd, type);
+        STACK_WIND (frame, dht_entrylk_cbk,
+                    subvol, subvol->fops->entrylk,
+                    volume, loc, basename, cmd, type, xdata);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
-static int32_t
+
+int
 dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-	DHT_STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, NULL);
+        return 0;
 }
 
-int32_t
+
+int
 dht_fentrylk (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, const char *basename,
-	      entrylk_cmd cmd, entrylk_type type)
+              const char *volume, fd_t *fd, const char *basename,
+              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        xlator_t     *subvol = NULL;
         int           op_errno = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        subvol = dht_subvol_get_cached (this, fd->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_fentrylk_cbk,
-		    subvol, subvol->fops->fentrylk,
-		    fd, basename, cmd, type);
+        STACK_WIND (frame, dht_fentrylk_cbk,
+                    subvol, subvol->fops->fentrylk,
+                    volume, fd, basename, cmd, type, xdata);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_forget (xlator_t *this, inode_t *inode)
 {
-	uint64_t      tmp_layout = 0;
-	dht_layout_t *layout = NULL;
+        uint64_t        ctx_int = 0;
+        dht_inode_ctx_t *ctx    = NULL;
+        dht_layout_t *layout = NULL;
 
-	inode_ctx_get (inode, this, &tmp_layout);
+        inode_ctx_del (inode, this, &ctx_int);
 
-	if (!layout)
-		return 0;
-	layout = (dht_layout_t *)(long)tmp_layout;
-	if (!layout->preset)
-		FREE (layout);
+        if (!ctx_int)
+                return 0;
 
-	return 0;
-}
+        ctx = (dht_inode_ctx_t *) (long) ctx_int;
 
+        layout = ctx->layout;
+        ctx->layout = NULL;
+        dht_layout_unref (this, layout);
+        GF_FREE (ctx);
 
+        return 0;
+}
 
-static int
-dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
+
+int
+dht_notify (xlator_t *this, int event, void *data, ...)
 {
-        xlator_list_t *subvols = NULL;
-        int            cnt = 0;
+        xlator_t                *subvol = NULL;
+        int                      cnt    = -1;
+        int                      i      = -1;
+        dht_conf_t              *conf   = NULL;
+        int                      ret    = -1;
+        int                      propagate = 0;
+
+        int                      had_heard_from_all = 0;
+        int                      have_heard_from_all = 0;
+        struct timeval           time = {0,};
+        gf_defrag_info_t        *defrag = NULL;
+        dict_t                  *dict   = NULL;
+        gf_defrag_type           cmd    = 0;
+        dict_t                  *output = NULL;
+        va_list                  ap;
+
+
+        conf = this->private;
+        if (!conf)
+                return ret;
+
+        /* had all subvolumes reported status once till now? */
+        had_heard_from_all = 1;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->last_event[i]) {
+                        had_heard_from_all = 0;
+                }
+        }
 
+        switch (event) {
+        case GF_EVENT_CHILD_UP:
+                subvol = data;
 
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                cnt++;
+                conf->gen++;
 
-        conf->subvolumes = CALLOC (cnt, sizeof (xlator_t *));
-        if (!conf->subvolumes) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "memory allocation failed :(");
-                return -1;
-        }
-        conf->subvolume_cnt = cnt;
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (subvol == conf->subvolumes[i]) {
+                                cnt = i;
+                                break;
+                        }
+                }
 
-        cnt = 0;
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                conf->subvolumes[cnt++] = subvols->xlator;
+                if (cnt == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "got GF_EVENT_CHILD_UP bad subvolume %s",
+                                subvol->name);
+                        break;
+                }
 
-	conf->subvolume_status = CALLOC (cnt, sizeof (char));
-	if (!conf->subvolume_status) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		return -1;
-	}
+                gettimeofday (&time, NULL);
+                LOCK (&conf->subvolume_lock);
+                {
+                        conf->subvolume_status[cnt] = 1;
+                        conf->last_event[cnt] = event;
+                        conf->subvol_up_time[cnt] = time.tv_sec;
+                }
+                UNLOCK (&conf->subvolume_lock);
 
-        return 0;
-}
+                /* one of the node came back up, do a stat update */
+                dht_get_du_info_for_subvol (this, cnt);
 
+                break;
 
-int
-dht_notify (xlator_t *this, int event, void *data, ...)
-{
-	xlator_t   *subvol = NULL;
-	int         cnt    = -1;
-	int         i      = -1;
-	dht_conf_t *conf   = NULL;
-	int         ret    = -1;
+        case GF_EVENT_CHILD_MODIFIED:
+                subvol = data;
 
+                conf->gen++;
+                propagate = 1;
 
-	conf = this->private;
+                break;
 
-	switch (event) {
-	case GF_EVENT_CHILD_UP:
-		subvol = data;
+        case GF_EVENT_CHILD_DOWN:
+                subvol = data;
 
-		conf->gen++;
+                if (conf->assert_no_child_down) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Received CHILD_DOWN. Exiting");
+                        if (conf->defrag) {
+                                gf_defrag_stop (conf->defrag, NULL);
+                        } else {
+                                kill (getpid(), SIGTERM);
+                        }
+                }
 
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (subvol == conf->subvolumes[i]) {
-				cnt = i;
-				break;
-			}
-		}
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (subvol == conf->subvolumes[i]) {
+                                cnt = i;
+                                break;
+                        }
+                }
 
-		if (cnt == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"got GF_EVENT_CHILD_UP bad subvolume %s",
-				subvol->name);
-			break;
-		}
+                if (cnt == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
+                                subvol->name);
+                        break;
+                }
 
-		LOCK (&conf->subvolume_lock);
-		{
-			conf->subvolume_status[cnt] = 1;
-		}
-		UNLOCK (&conf->subvolume_lock);
+                LOCK (&conf->subvolume_lock);
+                {
+                        conf->subvolume_status[cnt] = 0;
+                        conf->last_event[cnt] = event;
+                        conf->subvol_up_time[cnt] = 0;
+                }
+                UNLOCK (&conf->subvolume_lock);
 
-		break;
+                break;
 
-	case GF_EVENT_CHILD_DOWN:
-		subvol = data;
+        case GF_EVENT_CHILD_CONNECTING:
+                subvol = data;
 
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (subvol == conf->subvolumes[i]) {
-				cnt = i;
-				break;
-			}
-		}
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (subvol == conf->subvolumes[i]) {
+                                cnt = i;
+                                break;
+                        }
+                }
 
-		if (cnt == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"got GF_EVENT_CHILD_DOWN bad subvolume %s",
-				subvol->name);
-			break;
-		}
+                if (cnt == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "got GF_EVENT_CHILD_CONNECTING bad subvolume %s",
+                                subvol->name);
+                        break;
+                }
 
-		LOCK (&conf->subvolume_lock);
-		{
-			conf->subvolume_status[cnt] = 0;
-		}
-		UNLOCK (&conf->subvolume_lock);
+                LOCK (&conf->subvolume_lock);
+                {
+                        conf->last_event[cnt] = event;
+                }
+                UNLOCK (&conf->subvolume_lock);
+
+                break;
+        case GF_EVENT_VOLUME_DEFRAG:
+        {
+                if (!conf->defrag) {
+                        return ret;
+                }
+                defrag = conf->defrag;
+
+                dict = data;
+                va_start (ap, data);
+                output = va_arg (ap, dict_t*);
+
+                ret = dict_get_int32 (dict, "rebalance-command",
+                                      (int32_t*)&cmd);
+                if (ret)
+                        return ret;
+                LOCK (&defrag->lock);
+                {
+                        if (defrag->is_exiting)
+                                goto unlock;
+                        if (cmd == GF_DEFRAG_CMD_STATUS)
+                                gf_defrag_status_get (defrag, output);
+                        else if (cmd == GF_DEFRAG_CMD_STOP)
+                                gf_defrag_stop (defrag, output);
+                }
+unlock:
+                UNLOCK (&defrag->lock);
+                return 0;
+                break;
+        }
+
+        default:
+                propagate = 1;
+                break;
+        }
+
+
+        /* have all subvolumes reported status once by now? */
+        have_heard_from_all = 1;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->last_event[i])
+                        have_heard_from_all = 0;
+        }
+
+        /* if all subvols have reported status, no need to hide anything
+           or wait for anything else. Just propagate blindly */
+        if (have_heard_from_all) {
+                propagate = 1;
+
+        }
 
-		break;
-	}
 
-	ret = default_notify (this, event, data);
+        if (!had_heard_from_all && have_heard_from_all) {
+                /* This is the first event which completes aggregation
+                   of events from all subvolumes. If at least one subvol
+                   had come up, propagate CHILD_UP, but only this time
+                */
+                event = GF_EVENT_CHILD_DOWN;
 
-	return ret;
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
+                                event = GF_EVENT_CHILD_UP;
+                                break;
+                        }
+
+                        if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                                event = GF_EVENT_CHILD_CONNECTING;
+                                /* continue to check other events for CHILD_UP */
+                        }
+                }
+
+                /* rebalance is started with assert_no_child_down. So we do
+                 * not need to handle CHILD_DOWN event here.
+                 */
+                if (conf->defrag) {
+                        ret = gf_thread_create (&conf->defrag->th, NULL,
+						gf_defrag_start, this);
+                        if (ret) {
+                                conf->defrag = NULL;
+                                GF_FREE (conf->defrag);
+                                kill (getpid(), SIGTERM);
+                        }
+                }
+        }
+
+        ret = 0;
+        if (propagate)
+                ret = default_notify (this, event, data);
+
+        return ret;
 }
 
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (!ret && ctx) {
+                if (ctx->layout) {
+                        if (layout)
+                                *layout = ctx->layout;
+                        ret = 0;
+                } else {
+                        ret = -1;
+                }
+        }
+
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 8f0cbfb7b..5ccd66799 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -22,196 +13,775 @@
 #include "config.h"
 #endif
 
+#include <regex.h>
+
+#include "dht-mem-types.h"
+#include "libxlator.h"
+#include "syncop.h"
+
 #ifndef _DHT_H
 #define _DHT_H
 
+#define GF_XATTR_FIX_LAYOUT_KEY     "distribute.fix.layout"
+#define GF_DHT_LOOKUP_UNHASHED_ON   1
+#define GF_DHT_LOOKUP_UNHASHED_AUTO 2
+#define DHT_PATHINFO_HEADER         "DISTRIBUTE:"
+
+#include <fnmatch.h>
 
 typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
-				       xlator_t *this,
-				       int32_t op_ret, int32_t op_errno);
+                                       xlator_t     *this,
+                                       int32_t       op_ret, int32_t op_errno,
+                                       dict_t *xdata);
+typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, call_frame_t *frame,
+                                    int              ret);
 
 
 struct dht_layout {
-        int               cnt;
-	int               preset;
-        int               gen;
-	int               type;
+        int                spread_cnt;  /* layout spread count per directory,
+                                           is controlled by 'setxattr()' with
+                                           special key */
+        int                cnt;
+        int                preset;
+        int                gen;
+        int                type;
+        int                ref; /* use with dht_conf_t->layout_lock */
+        int                search_unhashed;
         struct {
-		int       err;   /* 0 = normal
-				   -1 = dir exists and no xattr
-				   >0 = dir lookup failed with errno
-				 */
-                uint32_t  start;
-                uint32_t  stop;
-                xlator_t *xlator;
-        } list[0];
+                int        err;   /* 0 = normal
+                                     -1 = dir exists and no xattr
+                                     >0 = dir lookup failed with errno
+                                  */
+                uint32_t   start;
+                uint32_t   stop;
+                xlator_t  *xlator;
+        } list[];
+};
+typedef struct dht_layout  dht_layout_t;
+
+struct dht_stat_time {
+        uint32_t        atime;
+        uint32_t        atime_nsec;
+        uint32_t        ctime;
+        uint32_t        ctime_nsec;
+        uint32_t        mtime;
+        uint32_t        mtime_nsec;
+};
+
+typedef struct dht_stat_time dht_stat_time_t;
+
+struct dht_inode_ctx {
+        dht_layout_t    *layout;
+        dht_stat_time_t  time;
 };
-typedef struct dht_layout dht_layout_t;
 
+typedef struct dht_inode_ctx dht_inode_ctx_t;
+
+
+typedef enum {
+        DHT_HASH_TYPE_DM,
+        DHT_HASH_TYPE_DM_USER,
+} dht_hashfn_type_t;
+
+/* rebalance related */
+struct dht_rebalance_ {
+        xlator_t            *from_subvol;
+        xlator_t            *target_node;
+        off_t                offset;
+        size_t               size;
+        int32_t              flags;
+        int                  count;
+        struct iobref       *iobref;
+        struct iovec        *vector;
+        struct iatt          stbuf;
+        dht_defrag_cbk_fn_t  target_op_fn;
+        dict_t              *xdata;
+};
 
 struct dht_local {
-	int                      call_cnt;
-	loc_t                    loc;
-	loc_t                    loc2;
-	int                      op_ret;
-	int                      op_errno;
-	int                      layout_mismatch;
-	struct stat              stbuf;
-	struct statvfs           statvfs;
-	fd_t                    *fd;
-	inode_t                 *inode;
-	dict_t                  *xattr;
-	dict_t                  *xattr_req;
-	dht_layout_t            *layout;
-	size_t                   size;
-	ino_t                    st_ino;
-	xlator_t                *src_hashed, *src_cached;
-	xlator_t                *dst_hashed, *dst_cached;
-	xlator_t                *cached_subvol;
-	xlator_t                *hashed_subvol;
-	char                     need_selfheal;
-	struct {
-		fop_mknod_cbk_t  linkfile_cbk;
-		struct stat      stbuf;
-		loc_t            loc;
-		inode_t         *inode;
-		dict_t          *xattr;
-		xlator_t        *srcvol;
-	} linkfile;
-	struct {
-		uint32_t         hole_cnt;
-		uint32_t         overlaps_cnt;
-		uint32_t         missing;
-		uint32_t         down;
-		uint32_t         misc;
-		dht_selfheal_dir_cbk_t   dir_cbk;
-		dht_layout_t    *layout;
-	} selfheal;
-
-	/* needed by nufa */
-	int32_t flags;
-	mode_t  mode;
-	dev_t   rdev;
+        int                      call_cnt;
+        loc_t                    loc;
+        loc_t                    loc2;
+        int                      op_ret;
+        int                      op_errno;
+        int                      layout_mismatch;
+        /* Use stbuf as the postbuf, when we require both
+         * pre and post attrs */
+        struct iatt              stbuf;
+        struct iatt              prebuf;
+        struct iatt              preoldparent;
+        struct iatt              postoldparent;
+        struct iatt              preparent;
+        struct iatt              postparent;
+        struct statvfs           statvfs;
+        fd_t                    *fd;
+        inode_t                 *inode;
+        dict_t                  *params;
+        dict_t                  *xattr;
+        dict_t                  *xattr_req;
+        dht_layout_t            *layout;
+        size_t                   size;
+        ino_t                    ia_ino;
+        xlator_t                *src_hashed, *src_cached;
+        xlator_t                *dst_hashed, *dst_cached;
+        xlator_t                *cached_subvol;
+        xlator_t                *hashed_subvol;
+        char                     need_selfheal;
+        int                      file_count;
+        int                      dir_count;
+        call_frame_t            *main_frame;
+        int                      fop_succeeded;
+        struct {
+                fop_mknod_cbk_t  linkfile_cbk;
+                struct iatt      stbuf;
+                loc_t            loc;
+                inode_t         *inode;
+                dict_t          *xattr;
+                xlator_t        *srcvol;
+        } linkfile;
+        struct {
+                uint32_t         hole_cnt;
+                uint32_t         overlaps_cnt;
+                uint32_t         down;
+                uint32_t         misc;
+                dht_selfheal_dir_cbk_t   dir_cbk;
+                dht_layout_t    *layout;
+        } selfheal;
+        uint32_t                 uid;
+        uint32_t                 gid;
+
+        /* needed by nufa */
+        int32_t flags;
+        mode_t  mode;
+        dev_t   rdev;
+        mode_t  umask;
+
+        /* need for file-info */
+        char   *xattr_val;
+        char   *key;
+
+        /* which xattr request? */
+        char xsel[256];
+        int32_t alloc_len;
+
+        char   *newpath;
+
+        /* gfid related */
+        uuid_t  gfid;
+
+        /*Marker Related*/
+        struct marker_str    marker;
+
+        /* flag used to make sure we need to return estale in
+           {lookup,revalidate}_cbk */
+        char return_estale;
+        char need_lookup_everywhere;
+
+        glusterfs_fop_t      fop;
+
+        gf_boolean_t     linked;
+        xlator_t        *link_subvol;
+
+        struct dht_rebalance_ rebalance;
+        xlator_t        *first_up_subvol;
+
 };
 typedef struct dht_local dht_local_t;
 
+/* du - disk-usage */
+struct dht_du {
+        double   avail_percent;
+	double   avail_inodes;
+        uint64_t avail_space;
+        uint32_t log;
+};
+typedef struct dht_du dht_du_t;
+
+enum gf_defrag_type {
+        GF_DEFRAG_CMD_START = 1,
+        GF_DEFRAG_CMD_STOP = 1 + 1,
+        GF_DEFRAG_CMD_STATUS = 1 + 2,
+        GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
+        GF_DEFRAG_CMD_START_FORCE = 1 + 4,
+};
+typedef enum gf_defrag_type gf_defrag_type;
+
+enum gf_defrag_status_t {
+        GF_DEFRAG_STATUS_NOT_STARTED,
+        GF_DEFRAG_STATUS_STARTED,
+        GF_DEFRAG_STATUS_STOPPED,
+        GF_DEFRAG_STATUS_COMPLETE,
+        GF_DEFRAG_STATUS_FAILED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
+};
+typedef enum gf_defrag_status_t gf_defrag_status_t;
+
+typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
+
+struct gf_defrag_pattern_list {
+        char                       path_pattern[256];
+        uint64_t                   size;
+        gf_defrag_pattern_list_t  *next;
+};
+
+struct gf_defrag_info_ {
+        uint64_t                     total_files;
+        uint64_t                     total_data;
+        uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
+        uint64_t                     skipped;
+        gf_lock_t                    lock;
+        int                          cmd;
+        pthread_t                    th;
+        gf_defrag_status_t           defrag_status;
+        struct rpc_clnt             *rpc;
+        uint32_t                     connected;
+        uint32_t                     is_exiting;
+        pid_t                        pid;
+        inode_t                     *root_inode;
+        uuid_t                       node_uuid;
+        struct timeval               start_time;
+        gf_boolean_t                 stats;
+        gf_defrag_pattern_list_t    *defrag_pattern;
+};
+
+typedef struct gf_defrag_info_ gf_defrag_info_t;
 
 struct dht_conf {
-	gf_lock_t      subvolume_lock;
+        gf_lock_t      subvolume_lock;
         int            subvolume_cnt;
         xlator_t     **subvolumes;
-	xlator_t      *local_volume;     /* Needed by NUFA */
-	char          *subvolume_status;
-	dht_layout_t **file_layouts;
-	dht_layout_t **dir_layouts;
-	dht_layout_t  *default_dir_layout;
-	gf_boolean_t   search_unhashed;
-	int            gen;
+        char          *subvolume_status;
+        int           *last_event;
+        dht_layout_t **file_layouts;
+        dht_layout_t **dir_layouts;
+        gf_boolean_t   search_unhashed;
+        int            gen;
+        dht_du_t      *du_stats;
+        double         min_free_disk;
+	double         min_free_inodes;
+        char           disk_unit;
+        int32_t        refresh_interval;
+        gf_boolean_t   unhashed_sticky_bit;
+        struct timeval last_stat_fetch;
+        gf_lock_t      layout_lock;
+        void          *private;     /* Can be used by wrapper xlators over
+                                       dht */
+        gf_boolean_t   use_readdirp;
+        char           vol_uuid[UUID_SIZE + 1];
+        gf_boolean_t   assert_no_child_down;
+        time_t        *subvol_up_time;
+
+        /* This is the count used as the distribute layout for a directory */
+        /* Will be a global flag to control the layout spread count */
+        uint32_t       dir_spread_cnt;
+
+        /* to keep track of nodes which are decomissioned */
+        xlator_t     **decommissioned_bricks;
+        int            decommission_in_progress;
+        int            decommission_subvols_cnt;
+
+        /* defrag related */
+        gf_defrag_info_t *defrag;
+
+        /* Request to filter directory entries in readdir request */
+
+        gf_boolean_t    readdir_optimize;
+
+        /* Support regex-based name reinterpretation. */
+        regex_t         rsync_regex;
+        gf_boolean_t    rsync_regex_valid;
+        regex_t         extra_regex;
+        gf_boolean_t    extra_regex_valid;
+
+        /* Support variable xattr names. */
+        char            *xattr_name;
+        char            *link_xattr_name;
+        char            *wild_xattr_name;
 };
 typedef struct dht_conf dht_conf_t;
 
 
 struct dht_disk_layout {
-	uint32_t           cnt;
-	uint32_t           type;
-	struct {
-		uint32_t   start;
-		uint32_t   stop;
-	} list[1];
+        uint32_t           cnt;
+        uint32_t           type;
+        struct {
+                uint32_t   start;
+                uint32_t   stop;
+        } list[1];
 };
 typedef struct dht_disk_layout dht_disk_layout_t;
- 
-#define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_fs_root(loc) (strcmp (loc->path, "/") == 0)
+typedef enum {
+        GF_DHT_MIGRATE_DATA,
+        GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
+        GF_DHT_MIGRATE_HARDLINK,
+        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
+} gf_dht_migrate_data_type_t;
+
+#define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_revalidate(loc) (inode_ctx_get (loc->inode, this, NULL) == 0)
+#define is_revalidate(loc) (dht_inode_ctx_layout_get (loc->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
-#define DHT_LINKFILE_MODE (S_ISVTX)
-#define check_is_linkfile(i,s,x) ((s->st_mode & ~S_IFMT) == DHT_LINKFILE_MODE)
+#define DHT_MIGRATION_IN_PROGRESS 1
+#define DHT_MIGRATION_COMPLETED   2
+
+#define DHT_LINKFILE_MODE        (S_ISVTX)
+
+#define check_is_linkfile(i,s,x,n) (                                      \
+                ((st_mode_from_ia ((s)->ia_prot, (s)->ia_type) & ~S_IFMT) \
+                 == DHT_LINKFILE_MODE) &&                                 \
+                dict_get (x, n))
+
+#define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
+                IA_ISREG ((buf)->ia_type) &&                            \
+                ((st_mode_from_ia ((buf)->ia_prot, (buf)->ia_type) &    \
+                  ~S_IFMT) == DHT_LINKFILE_MODE))
+
+#define IS_DHT_MIGRATION_PHASE1(buf)  (                                 \
+                IA_ISREG ((buf)->ia_type) &&                            \
+                ((buf)->ia_prot.sticky == 1) &&                         \
+                ((buf)->ia_prot.sgid == 1))
+
+#define DHT_STRIP_PHASE1_FLAGS(buf)  do {                       \
+                if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {    \
+                        (buf)->ia_prot.sticky = 0;              \
+                        (buf)->ia_prot.sgid = 0;                \
+                }                                               \
+        } while (0)
 
-#define check_is_dir(i,s,x) (S_ISDIR(s->st_mode))
+#define check_is_dir(i,s,x) (IA_ISDIR(s->ia_type))
 
 #define layout_is_sane(layout) ((layout) && (layout->cnt > 0))
 
-#define DHT_STACK_UNWIND(frame, params ...) do {       \
-		dht_local_t *__local = NULL;           \
-		__local = frame->local;                \
-		frame->local = NULL;		       \
-		STACK_UNWIND (frame, params);          \
-		dht_local_wipe (__local);	       \
-	} while (0)
-
-#define DHT_STACK_DESTROY(frame) do {		       \
-		dht_local_t *__local = NULL;           \
-		__local = frame->local;                \
-		frame->local = NULL;		       \
-		STACK_DESTROY (frame->root);	       \
-		dht_local_wipe (__local);	       \
-	} while (0)
-
-dht_layout_t *dht_layout_new (xlator_t *this, int cnt);
-dht_layout_t *dht_layout_get (xlator_t *this, inode_t *inode);
-dht_layout_t *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_layout_search (xlator_t *this, dht_layout_t *layout,
-			     const char *name);
-int dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout);
-int dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
-			  uint32_t *holes_p, uint32_t *overlaps_p,
-			  uint32_t *missing_p, uint32_t *down_p,
-			  uint32_t *misc_p);
-int dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout,
-			     xlator_t *subvol, loc_t *loc, dict_t *xattr);
+#define DHT_STACK_UNWIND(fop, frame, params ...) do {           \
+                dht_local_t *__local = NULL;                    \
+                xlator_t    *__xl    = NULL;                    \
+                if (frame) {                                    \
+                        __xl         = frame->this;             \
+                        __local      = frame->local;            \
+                        frame->local = NULL;                    \
+                }                                               \
+                STACK_UNWIND_STRICT (fop, frame, params);       \
+                dht_local_wipe (__xl, __local);                 \
+        } while (0)
+
+#define DHT_STACK_DESTROY(frame) do {                   \
+                dht_local_t *__local = NULL;            \
+                xlator_t    *__xl    = NULL;            \
+                __xl                 = frame->this;     \
+                __local              = frame->local;    \
+                frame->local         = NULL;            \
+                STACK_DESTROY (frame->root);            \
+                dht_local_wipe (__xl, __local);         \
+        } while (0)
+
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, inode, post) do {\
+                int32_t sec = 0;                                        \
+                sec = new_sec;                                          \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        new_sec = max(new_sec, ctx_sec);                \
+                        if (sec < new_sec)                              \
+                                new_nsec = ctx_nsec;                    \
+                        if (sec == new_sec)                             \
+                                new_nsec = max (new_nsec, ctx_nsec);    \
+                        if (post) {                                     \
+                                ctx_sec = new_sec;                      \
+                                ctx_nsec = new_nsec;                    \
+                        }                                               \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
+dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
+dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
+dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_layout_search (xlator_t   *this, dht_layout_t *layout,
+                             const char *name);
+int                                      dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout);
+int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
+                          uint32_t      *holes_p, uint32_t *overlaps_p,
+                          uint32_t      *missing_p, uint32_t *down_p,
+                          uint32_t      *misc_p, uint32_t *no_space_p);
+int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
+                             xlator_t   *subvol, loc_t *loc, dict_t *xattr);
 
 xlator_t *dht_linkfile_subvol (xlator_t *this, inode_t *inode,
-			       struct stat *buf, dict_t *xattr);
-int dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
-			 xlator_t *subvol, loc_t *loc);
+                               struct iatt *buf, dict_t *xattr);
+int dht_linkfile_unlink (call_frame_t      *frame, xlator_t *this,
+                         xlator_t          *subvol, loc_t *loc);
 
 int dht_layouts_init (xlator_t *this, dht_conf_t *conf);
 int dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-		      int op_ret, int op_errno, dict_t *xattr);
+                      int       op_ret, int op_errno, dict_t *xattr);
 
 int dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-			     int pos, int32_t **disk_layout_p);
-int dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
-			   int pos, int32_t *disk_layout);
+                             int       pos, int32_t **disk_layout_p);
+int dht_disk_layout_merge (xlator_t   *this, dht_layout_t *layout,
+                           int         pos, void *disk_layout_raw, int disk_layout_len);
 
 
 int dht_frame_return (call_frame_t *frame);
 
-int dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y);
+int                             dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y);
 int dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol,
-		      uint64_t *x);
+                      uint64_t *x);
 
-void dht_local_wipe (dht_local_t *local);
-dht_local_t *dht_local_init (call_frame_t *frame);
-int dht_stat_merge (xlator_t *this, struct stat *to, struct stat *from,
-		    xlator_t *subvol);
+void dht_local_wipe (xlator_t *this, dht_local_t *local);
+dht_local_t *dht_local_init (call_frame_t    *frame, loc_t *loc, fd_t *fd,
+                             glusterfs_fop_t  fop);
+int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt *from,
+                    xlator_t                 *subvol);
 
 xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
 xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
 xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
-int dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_subvol_next_available (xlator_t *this, xlator_t *prev);
+int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
 
-int dht_hash_compute (int type, const char *name, uint32_t *hash_p);
+int dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p);
 
-int dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
-			 xlator_t *tovol, xlator_t *fromvol, loc_t *loc);
+int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
+                         xlator_t        *this, xlator_t *tovol,
+                         xlator_t        *fromvol, loc_t *loc);
+int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
+int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int
-dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-			loc_t *loc, dht_layout_t *layout);
+dht_selfheal_directory (call_frame_t     *frame, dht_selfheal_dir_cbk_t cbk,
+                        loc_t            *loc, dht_layout_t *layout);
 int
 dht_selfheal_new_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-			    dht_layout_t *layout);
+                            dht_layout_t *layout);
 int
-dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-		      loc_t *loc, dht_layout_t *layout);
+dht_selfheal_restore (call_frame_t       *frame, dht_selfheal_dir_cbk_t cbk,
+                      loc_t              *loc, dht_layout_t *layout);
 int
 dht_layout_sort_volname (dht_layout_t *layout);
 
-int dht_rename (call_frame_t *frame, xlator_t *this,
-		loc_t *oldloc, loc_t *newloc);
-#endif /* _DHT_H */
+int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
+
+gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                          dht_local_t *layout);
+int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
+
+int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
+int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);;
+void          dht_layout_unref (xlator_t *this, dht_layout_t *layout);
+dht_layout_t *dht_layout_ref (xlator_t *this, dht_layout_t *layout);
+xlator_t     *dht_first_up_subvol (xlator_t *this);
+xlator_t     *dht_last_up_subvol (xlator_t *this);
+
+int dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
+
+int dht_filter_loc_subvol_key (xlator_t  *this, loc_t *loc, loc_t *new_loc,
+                               xlator_t **subvol);
+
+int                                     dht_rename_cleanup (call_frame_t *frame);
+int dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t           op_ret, int32_t op_errno,
+                          inode_t          *inode, struct iatt *stbuf,
+                          struct iatt      *preparent, struct iatt *postparent,
+                          dict_t *xdata);
+
+int dht_fix_directory_layout (call_frame_t *frame,
+                              dht_selfheal_dir_cbk_t  dir_cbk,
+                              dht_layout_t           *layout);
+
+int dht_init_subvolumes (xlator_t *this, dht_conf_t *conf);
+
+/* migration/rebalance */
+int dht_start_rebalance_task (xlator_t *this, call_frame_t *frame);
+
+int dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame);
+int dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame);
+
+
+/* FOPS */
+int32_t dht_lookup (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *loc,
+                    dict_t   *xattr_req);
+
+int32_t dht_stat (call_frame_t *frame,
+                  xlator_t *this,
+                  loc_t    *loc, dict_t *xdata);
+
+int32_t dht_fstat (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t     *fd, dict_t *xdata);
+
+int32_t dht_truncate (call_frame_t *frame,
+                      xlator_t *this,
+                      loc_t    *loc,
+                      off_t     offset, dict_t *xdata);
+
+int32_t dht_ftruncate (call_frame_t *frame,
+                       xlator_t *this,
+                       fd_t     *fd,
+                       off_t     offset, dict_t *xdata);
+
+int32_t dht_access (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *loc,
+                    int32_t   mask, dict_t *xdata);
+
+int32_t dht_readlink (call_frame_t *frame,
+                      xlator_t *this,
+                      loc_t    *loc,
+                      size_t    size, dict_t *xdata);
+
+int32_t dht_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
+
+int32_t dht_mkdir (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
+
+int32_t dht_unlink (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *loc, int xflag, dict_t *xdata);
+
+int32_t dht_rmdir (call_frame_t *frame, xlator_t *this,
+                   loc_t    *loc, int flags, dict_t *xdata);
+
+int32_t dht_symlink (call_frame_t   *frame, xlator_t *this,
+                     const char *linkpath, loc_t *loc, mode_t umask,
+                     dict_t *xdata);
+
+int32_t dht_rename (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *oldloc,
+                    loc_t    *newloc, dict_t *xdata);
+
+int32_t dht_link (call_frame_t *frame,
+                  xlator_t *this,
+                  loc_t    *oldloc,
+                  loc_t    *newloc, dict_t *xdata);
+
+int32_t dht_create (call_frame_t *frame, xlator_t *this,
+                    loc_t    *loc, int32_t flags, mode_t mode,
+                    mode_t umask, fd_t     *fd, dict_t *params);
+
+int32_t dht_open (call_frame_t *frame,
+                  xlator_t *this,
+                  loc_t    *loc,
+                  int32_t   flags, fd_t *fd, dict_t *xdata);
+
+int32_t dht_readv (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t     *fd,
+                   size_t    size,
+                   off_t     offset, uint32_t flags, dict_t *xdata);
+
+int32_t dht_writev (call_frame_t      *frame,
+                    xlator_t      *this,
+                    fd_t          *fd,
+                    struct iovec  *vector,
+                    int32_t        count,
+                    off_t          offset,
+                    uint32_t       flags,
+                    struct iobref *iobref, dict_t *xdata);
+
+int32_t dht_flush (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t     *fd, dict_t *xdata);
+
+int32_t dht_fsync (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t     *fd,
+                   int32_t   datasync, dict_t *xdata);
+
+int32_t dht_opendir (call_frame_t *frame,
+                     xlator_t *this,
+                     loc_t    *loc, fd_t *fd, dict_t *xdata);
+
+int32_t dht_fsyncdir (call_frame_t *frame,
+                      xlator_t *this,
+                      fd_t     *fd,
+                      int32_t   datasync, dict_t *xdata);
+
+int32_t dht_statfs (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *loc, dict_t *xdata);
+
+int32_t dht_setxattr (call_frame_t *frame,
+                      xlator_t *this,
+                      loc_t    *loc,
+                      dict_t   *dict,
+                      int32_t   flags, dict_t *xdata);
+
+int32_t dht_getxattr (call_frame_t   *frame,
+                      xlator_t   *this,
+                      loc_t      *loc,
+                      const char *name, dict_t *xdata);
+
+int32_t dht_fsetxattr (call_frame_t *frame,
+                       xlator_t *this,
+                       fd_t     *fd,
+                       dict_t   *dict,
+                       int32_t   flags, dict_t *xdata);
+
+int32_t dht_fgetxattr (call_frame_t   *frame,
+                       xlator_t   *this,
+                       fd_t       *fd,
+                       const char *name, dict_t *xdata);
+
+int32_t dht_removexattr (call_frame_t   *frame,
+                         xlator_t   *this,
+                         loc_t      *loc,
+                         const char *name, dict_t *xdata);
+int32_t dht_fremovexattr (call_frame_t   *frame,
+                          xlator_t   *this,
+                          fd_t      *fd,
+                          const char *name, dict_t *xdata);
+
+int32_t dht_lk (call_frame_t        *frame,
+                xlator_t        *this,
+                fd_t            *fd,
+                int32_t          cmd,
+                struct gf_flock *flock, dict_t *xdata);
+
+int32_t dht_inodelk (call_frame_t *frame, xlator_t *this,
+                     const char      *volume, loc_t *loc, int32_t cmd,
+                     struct gf_flock *flock, dict_t *xdata);
+
+int32_t dht_finodelk (call_frame_t        *frame, xlator_t *this,
+                      const char      *volume, fd_t *fd, int32_t cmd,
+                      struct gf_flock *flock, dict_t *xdata);
+
+int32_t dht_entrylk (call_frame_t    *frame, xlator_t *this,
+                     const char  *volume, loc_t *loc, const char *basename,
+                     entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
+
+int32_t dht_fentrylk (call_frame_t    *frame, xlator_t *this,
+                      const char  *volume, fd_t *fd, const char *basename,
+                      entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
+
+int32_t dht_readdir (call_frame_t  *frame,
+                     xlator_t *this,
+                     fd_t     *fd,
+                     size_t    size, off_t off, dict_t *xdata);
+
+int32_t dht_readdirp (call_frame_t *frame,
+                      xlator_t *this,
+                      fd_t     *fd,
+                      size_t    size, off_t off, dict_t *dict);
+
+int32_t dht_xattrop (call_frame_t           *frame,
+                     xlator_t           *this,
+                     loc_t              *loc,
+                     gf_xattrop_flags_t  flags,
+                     dict_t             *dict, dict_t *xdata);
+
+int32_t dht_fxattrop (call_frame_t *frame,
+                      xlator_t           *this,
+                      fd_t               *fd,
+                      gf_xattrop_flags_t  flags,
+                      dict_t             *dict, dict_t *xdata);
+
+int32_t dht_forget (xlator_t *this, inode_t *inode);
+int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
+                     struct iatt   *stbuf, int32_t valid, dict_t *xdata);
+int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      struct iatt  *stbuf, int32_t valid, dict_t *xdata);
+int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      int32_t mode, off_t offset, size_t len, dict_t *xdata);
+int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		    off_t offset, size_t len, dict_t *xdata);
+int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    off_t offset, size_t len, dict_t *xdata);
+
+int32_t dht_init (xlator_t *this);
+void    dht_fini (xlator_t *this);
+int     dht_reconfigure (xlator_t *this, dict_t *options);
+int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
+
+/* definitions for nufa/switch */
+int dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent);
+int dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent);
+int dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
+                             xlator_t *this, int op_ret, int op_errno,
+                             inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                             struct iatt *postparent);
+int dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno,
+                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                    struct iatt *postparent);
+int dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno,
+                    fd_t *fd, inode_t *inode, struct iatt *stbuf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t *xdata);
+int dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno,
+                     inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata);
+
+int
+gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
+
+int
+gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output);
+
+void*
+gf_defrag_start (void *this);
+
+int32_t
+gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
+                           struct iatt *stbuf);
+int
+dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag);
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,
+                          dht_layout_t **layout_int);
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t* layout_int);
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t update_ctx);
+
+int dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
+int
+dht_dir_attr_heal (void *data);
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data);
+int
+dht_dir_has_layout (dict_t *xattr, char *name);
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator);
+xlator_t *
+dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol,
+                                   dht_layout_t *layout);
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout);
+int
+dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this);
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix);
+int32_t
+dht_priv_dump (xlator_t *this);
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode);
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol);
+
+#endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
new file mode 100644
index 000000000..fe3955ecb
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -0,0 +1,414 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+/* TODO: add NS locking */
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "dht-common.h"
+#include "defaults.h"
+
+#include <sys/time.h>
+
+
+int
+dht_du_info_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int op_ret, int op_errno, struct statvfs *statvfs,
+                 dict_t *xdata)
+{
+	dht_conf_t    *conf         = NULL;
+	call_frame_t  *prev          = NULL;
+	int            this_call_cnt = 0;
+	int            i = 0;
+	double         percent = 0;
+	double         percent_inodes = 0;
+	uint64_t       bytes = 0;
+
+	conf = this->private;
+	prev = cookie;
+
+	if (op_ret == -1) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"failed to get disk info from %s", prev->this->name);
+		goto out;
+	}
+
+	if (statvfs && statvfs->f_blocks) {
+		percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
+		bytes = (statvfs->f_bavail * statvfs->f_frsize);
+	}
+
+	if (statvfs && statvfs->f_files) {
+		percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
+	} else {
+		/* set percent inodes to 100 for dynamically allocated inode filesystems
+		   this logic holds good so that, distribute has nothing to worry about
+		   total inodes rather let the 'create()' to be scheduled on the hashed
+		   subvol regardless of the total inodes. since we have no awareness on
+		   loosing inodes this logic fits well
+		*/
+		percent_inodes = 100;
+	}
+
+	LOCK (&conf->subvolume_lock);
+	{
+		for (i = 0; i < conf->subvolume_cnt; i++)
+			if (prev->this == conf->subvolumes[i]) {
+				conf->du_stats[i].avail_percent = percent;
+				conf->du_stats[i].avail_space   = bytes;
+				conf->du_stats[i].avail_inodes  = percent_inodes;
+				gf_log (this->name, GF_LOG_DEBUG,
+					"on subvolume '%s': avail_percent is: "
+					"%.2f and avail_space is: %"PRIu64" "
+					"and avail_inodes is: %.2f",
+					prev->this->name,
+					conf->du_stats[i].avail_percent,
+					conf->du_stats[i].avail_space,
+					conf->du_stats[i].avail_inodes);
+			}
+	}
+	UNLOCK (&conf->subvolume_lock);
+
+out:
+	this_call_cnt = dht_frame_return (frame);
+	if (is_last_call (this_call_cnt))
+		DHT_STACK_DESTROY (frame);
+
+	return 0;
+}
+
+int
+dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
+{
+	dht_conf_t    *conf         = NULL;
+	call_frame_t  *statfs_frame = NULL;
+	dht_local_t   *statfs_local = NULL;
+	call_pool_t   *pool         = NULL;
+	loc_t          tmp_loc      = {0,};
+
+	conf = this->private;
+	pool = this->ctx->pool;
+
+	statfs_frame = create_frame (this, pool);
+	if (!statfs_frame) {
+		goto err;
+	}
+
+	/* local->fop value is not used in this case */
+	statfs_local = dht_local_init (statfs_frame, NULL, NULL,
+				       GF_FOP_MAXVALUE);
+	if (!statfs_local) {
+		goto err;
+	}
+
+        /* make it root gfid, should be enough to get the proper info back */
+        tmp_loc.gfid[15] = 1;
+
+	statfs_local->call_cnt = 1;
+	STACK_WIND (statfs_frame, dht_du_info_cbk,
+		    conf->subvolumes[subvol_idx],
+		    conf->subvolumes[subvol_idx]->fops->statfs,
+		    &tmp_loc, NULL);
+
+	return 0;
+err:
+	if (statfs_frame)
+		DHT_STACK_DESTROY (statfs_frame);
+
+	return -1;
+}
+
+int
+dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+	int            i            = 0;
+	dht_conf_t    *conf         = NULL;
+	call_frame_t  *statfs_frame = NULL;
+	dht_local_t   *statfs_local = NULL;
+	struct timeval tv           = {0,};
+        loc_t          tmp_loc      = {0,};
+
+	conf  = this->private;
+
+	gettimeofday (&tv, NULL);
+
+        /* make it root gfid, should be enough to get the proper
+           info back */
+        tmp_loc.gfid[15] = 1;
+
+	if (tv.tv_sec > (conf->refresh_interval
+			 + conf->last_stat_fetch.tv_sec)) {
+
+		statfs_frame = copy_frame (frame);
+		if (!statfs_frame) {
+			goto err;
+		}
+
+		/* In this case, 'local->fop' is not used */
+		statfs_local = dht_local_init (statfs_frame, loc, NULL,
+					       GF_FOP_MAXVALUE);
+		if (!statfs_local) {
+			goto err;
+		}
+
+		statfs_local->call_cnt = conf->subvolume_cnt;
+		for (i = 0; i < conf->subvolume_cnt; i++) {
+			STACK_WIND (statfs_frame, dht_du_info_cbk,
+				    conf->subvolumes[i],
+				    conf->subvolumes[i]->fops->statfs,
+				    &tmp_loc, NULL);
+		}
+
+		conf->last_stat_fetch.tv_sec = tv.tv_sec;
+	}
+	return 0;
+err:
+	if (statfs_frame)
+		DHT_STACK_DESTROY (statfs_frame);
+
+	return -1;
+}
+
+
+gf_boolean_t
+dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
+{
+	int         i = 0;
+	dht_conf_t *conf = NULL;
+	gf_boolean_t subvol_filled_inodes = _gf_false;
+	gf_boolean_t subvol_filled_space = _gf_false;
+	gf_boolean_t is_subvol_filled = _gf_false;
+
+	conf = this->private;
+
+	/* Check for values above specified percent or free disk */
+	LOCK (&conf->subvolume_lock);
+	{
+		for (i = 0; i < conf->subvolume_cnt; i++) {
+			if (subvol == conf->subvolumes[i]) {
+				if (conf->disk_unit == 'p') {
+					if (conf->du_stats[i].avail_percent <
+					    conf->min_free_disk) {
+						subvol_filled_space = _gf_true;
+						break;
+					}
+
+				} else {
+					if (conf->du_stats[i].avail_space <
+					    conf->min_free_disk) {
+						subvol_filled_space = _gf_true;
+						break;
+					}
+				}
+				if (conf->du_stats[i].avail_inodes <
+				    conf->min_free_inodes) {
+					subvol_filled_inodes = _gf_true;
+					break;
+				}
+			}
+		}
+	}
+	UNLOCK (&conf->subvolume_lock);
+
+	if (subvol_filled_space && conf->subvolume_status[i]) {
+		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+			gf_log (this->name, GF_LOG_WARNING,
+				"disk space on subvolume '%s' is getting "
+				"full (%.2f %%), consider adding more nodes",
+				subvol->name,
+				(100 - conf->du_stats[i].avail_percent));
+		}
+	}
+
+	if (subvol_filled_inodes && conf->subvolume_status[i]) {
+		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+			gf_log (this->name, GF_LOG_CRITICAL,
+				"inodes on subvolume '%s' are at "
+				"(%.2f %%), consider adding more nodes",
+				subvol->name,
+				(100 - conf->du_stats[i].avail_inodes));
+		}
+	}
+
+	is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
+
+	return is_subvol_filled;
+}
+
+
+/*Get the best subvolume to create the file in*/
+xlator_t *
+dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                dht_local_t *local)
+{
+	xlator_t   *avail_subvol = NULL;
+	dht_conf_t *conf = NULL;
+        dht_layout_t *layout = NULL;
+        loc_t      *loc = NULL;
+
+	conf = this->private;
+        if (!local)
+                goto out;
+        loc = &local->loc;
+        if (!local->layout) {
+                layout = dht_layout_get (this, loc->parent);
+
+                if (!layout) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "layout missing path=%s parent=%s",
+                                loc->path, uuid_utoa (loc->parent->gfid));
+                        goto out;
+                }
+        } else {
+                layout = dht_layout_ref (this, local->layout);
+        }
+
+        LOCK (&conf->subvolume_lock);
+	{
+                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol,
+                                                                 layout);
+                if(!avail_subvol)
+                {
+                        avail_subvol = dht_subvol_maxspace_nonzeroinode(this,
+                                                                        subvol,
+                                                                        layout);
+                }
+
+	}
+	UNLOCK (&conf->subvolume_lock);
+out:
+	if (!avail_subvol) {
+		gf_log (this->name,
+                        GF_LOG_DEBUG,
+			"no subvolume has enough free space and/or inodes\
+                         to create");
+                avail_subvol = subvol;
+	}
+
+        if (layout)
+                dht_layout_unref (this, layout);
+	return avail_subvol;
+}
+
+static inline
+int32_t dht_subvol_has_err (xlator_t *this, dht_layout_t *layout)
+{
+        int ret = -1;
+        int i   = 0;
+
+        if (!this || !layout)
+                goto out;
+
+        /* check if subvol has layout errors, before selecting it */
+        for (i = 0; i < layout->cnt; i++) {
+                if (!strcmp (layout->list[i].xlator->name, this->name) &&
+                     (layout->list[i].err != 0)) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+/*Get subvolume which has both space and inodes more than the min criteria*/
+xlator_t *
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
+{
+        int i = 0;
+        double max = 0;
+        double max_inodes = 0;
+        int    ignore_subvol = 0;
+
+        xlator_t *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        for(i=0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if ((conf->disk_unit == 'p') &&
+                    (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_percent > max)) {
+                                max = conf->du_stats[i].avail_percent;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+                }
+
+                if ((conf->disk_unit != 'p') &&
+                    (conf->du_stats[i].avail_space > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_space > max)) {
+                                max = conf->du_stats[i].avail_space;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+                }
+        }
+
+        return avail_subvol;
+}
+
+
+/* Get subvol which has atleast one inode and maximum space */
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
+{
+        int         i = 0;
+        double      max = 0;
+        int         ignore_subvol = 0;
+
+        xlator_t   *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if (conf->disk_unit == 'p') {
+                        if ((conf->du_stats[i].avail_percent > max)
+                            && (conf->du_stats[i].avail_inodes > 0 )) {
+                                max = conf->du_stats[i].avail_percent;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+               } else {
+                         if ((conf->du_stats[i].avail_space > max)
+                            && (conf->du_stats[i].avail_inodes > 0)) {
+                                 max = conf->du_stats[i].avail_space;
+                                 avail_subvol = conf->subvolumes[i];
+                         }
+               }
+        }
+
+        return avail_subvol;
+}
diff --git a/xlators/cluster/dht/src/dht-hashfn-tea.c b/xlators/cluster/dht/src/dht-hashfn-tea.c
deleted file mode 100644
index 248567a38..000000000
--- a/xlators/cluster/dht/src/dht-hashfn-tea.c
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#include <stdint.h>
-#include <stdio.h>
-#include <string.h>
-
-
-#define DELTA 0x9E3779B9
-#define FULLROUNDS 10		/* 32 is overkill, 16 is strong crypto */
-#define PARTROUNDS 6		/* 6 gets complete mixing */
-
-
-static int
-tearound (int rounds, uint32_t *array, uint32_t *h0, uint32_t *h1)
-{
-	uint32_t sum = 0;
-	int      n = 0;
-	uint32_t b0  = 0;
-	uint32_t b1  = 0;
-
-	b0 = *h0;
-	b1 = *h1;
-
-	n = rounds;
-
-	do {
-		sum += DELTA;
-		b0  += ((b1 << 4) + array[0])
-			^ (b1 + sum)
-			^ ((b1 >> 5) + array[1]);
-		b1  += ((b0 << 4) + array[2])
-			^ (b0 + sum)
-			^ ((b0 >> 5) + array[3]);
-	} while (--n);
-
-	*h0 += b0;
-	*h1 += b1;
-
-	return 0;
-}
-
-
-uint32_t
-__pad (int len)
-{
-	uint32_t pad = 0;
-
-	pad = (uint32_t) len | ((uint32_t) len << 8);
-	pad |= pad << 16;
-
-	return pad;
-}
-
-
-uint32_t
-dht_hashfn_tea (const char *msg, int len)
-{
-	uint32_t  h0 = 0x9464a485;
-	uint32_t  h1 = 0x542e1a94;
-	uint32_t  array[4];
-	uint32_t  pad = 0;
-	int       i = 0;
-	int       j = 0;
-	int       full_quads = 0;
-	int       full_words = 0;
-	int       full_bytes = 0;
-	uint32_t *intmsg = NULL;
-	int       word = 0;
-
-
-	intmsg = (uint32_t *) msg;
-	pad = __pad (len);
-
-	full_bytes   = len;
-	full_words   = len / 4;
-	full_quads   = len / 16;
-
-	for (i = 0; i < full_quads; i++) {
-		for (j = 0; j < 4; j++) {
-			word     = *intmsg;
-			array[j] = word;
-			intmsg++;
-			full_words--;
-			full_bytes -= 4;
-		}
-		tearound (PARTROUNDS, &array[0], &h0, &h1);
-	}
-
-	for (j = 0; j < 4; j++) {
-		if (full_words) {
-			word     = *intmsg;
-			array[j] = word;
-			intmsg++;
-			full_words--;
-			full_bytes -= 4;
-		} else {
-			array[j] = pad;
-			while (full_bytes) {
-				array[j] <<= 8;
-				array[j] |= msg[len - full_bytes];
-				full_bytes--;
-			}
-		}
-	}
-	tearound (FULLROUNDS, &array[0], &h0, &h1);
-
-	return h0 ^ h1;
-}
-
-
-#if 0
-int
-main (int argc, char *argv[])
-{
-	int i = 0;
-	int hashval = 0;
-
-	for (i = 1; i < argc; i++) {
-		hashval = tea (argv[i], strlen (argv[i]));
-		printf ("%s: %x\n", argv[i], hashval);
-	}
-}
-#endif
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index 9e321a43c..656cf23a0 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -26,63 +17,95 @@
 #include "glusterfs.h"
 #include "xlator.h"
 #include "dht-common.h"
-
-
-uint32_t dht_hashfn_tea (const char *name, int len);
-
-
-typedef enum {
-	DHT_HASH_TYPE_TEA,
-} dht_hashfn_type_t;
+#include "hashfn.h"
 
 
 int
 dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 {
-	int      ret = 0;
-	uint32_t hash = 0;
-
-	switch (type) {
-	case DHT_HASH_TYPE_TEA:
-		hash = dht_hashfn_tea (name, strlen (name));
-		break;
-	default:
-		ret = -1;
-		break;
-	}
-
-	if (ret == 0) {
-		*hash_p = hash;
-	}
-
-	return ret;
+        int      ret = 0;
+        uint32_t hash = 0;
+
+        switch (type) {
+        case DHT_HASH_TYPE_DM:
+        case DHT_HASH_TYPE_DM_USER:
+                hash = gf_dm_hashfn (name, strlen (name));
+                break;
+        default:
+                ret = -1;
+                break;
+        }
+
+        if (ret == 0) {
+                *hash_p = hash;
+        }
+
+        return ret;
 }
 
 
-#define MAKE_RSYNC_FRIENDLY_NAME(rsync_frndly_name, name) do {          \
-                rsync_frndly_name = (char *) name;			\
-                if (name[0] == '.') {                                   \
-                        char *dot   = 0;                                \
-                        int namelen = 0;                                \
-                                                                        \
-                        dot = strrchr (name, '.');                      \
-                        if (dot && dot > (name + 1) && *(dot + 1)) {    \
-                                namelen = (dot - name);                 \
-                                rsync_frndly_name = alloca (namelen);   \
-                                strncpy (rsync_frndly_name, name + 1,   \
-                                         namelen);                      \
-                                rsync_frndly_name[namelen - 1] = 0;     \
-                        }                                               \
-                }                                                       \
-        } while (0);
-
+static inline
+gf_boolean_t
+dht_munge_name (const char *original, char *modified, size_t len, regex_t *re)
+{
+        regmatch_t      matches[2];
+        size_t          new_len;
+
+        if (regexec(re,original,2,matches,0) != REG_NOMATCH) {
+                if (matches[1].rm_so != -1) {
+                        new_len = matches[1].rm_eo - matches[1].rm_so;
+                        /* Equal would fail due to the NUL at the end. */
+                        if (new_len < len) {
+                                memcpy (modified,original+matches[1].rm_so,
+                                        new_len);
+                                modified[new_len] = '\0';
+                                return _gf_true;
+                        }
+                }
+        }
+
+        /* This is guaranteed safe because of how the dest was allocated. */
+        strcpy(modified,original);
+        return _gf_false;
+}
 
 int
-dht_hash_compute (int type, const char *name, uint32_t *hash_p)
+dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-	char     *rsync_friendly_name = NULL;
-
-	MAKE_RSYNC_FRIENDLY_NAME (rsync_friendly_name, name);
-
-	return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
+        char            *rsync_friendly_name    = NULL;
+        dht_conf_t      *priv                   = this->private;
+        size_t           len                    = 0;
+        gf_boolean_t     munged                 = _gf_false;
+
+        /*
+         * It wouldn't be safe to use alloca in an inline function that doesn't
+         * actually get inlined, and it wouldn't be efficient to do a real
+         * allocation, so we use alloca here (if needed) and pass that to the
+         * inline.
+         */
+
+        if (priv->extra_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->extra_regex);
+        }
+
+        if (!munged && priv->rsync_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                gf_log (this->name, GF_LOG_TRACE, "trying regex for %s", name);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->rsync_regex);
+                if (munged) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "munged down to %s", rsync_friendly_name);
+                }
+        }
+
+        if (!munged) {
+                rsync_friendly_name = (char *)name;
+        }
+
+        return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index 52d072002..311a48112 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -27,177 +18,406 @@
 #include "xlator.h"
 #include "dht-common.h"
 
+static inline int
+dht_inode_ctx_set1 (xlator_t *this, inode_t *inode, xlator_t *subvol)
+{
+        uint64_t tmp_subvol = 0;
+
+        tmp_subvol = (long)subvol;
+        return inode_ctx_set1 (inode, this, &tmp_subvol);
+}
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol)
+{
+        int ret = -1;
+        uint64_t tmp_subvol = 0;
+
+        ret =  inode_ctx_get1 (inode, this, &tmp_subvol);
+        if (tmp_subvol && subvol)
+                *subvol = (xlator_t *)tmp_subvol;
+
+        return ret;
+}
+
 
 int
 dht_frame_return (call_frame_t *frame)
 {
-	dht_local_t *local = NULL;
-	int          this_call_cnt = -1;
+        dht_local_t *local = NULL;
+        int          this_call_cnt = -1;
+
+        if (!frame)
+                return -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                this_call_cnt = --local->call_cnt;
+        }
+        UNLOCK (&frame->lock);
+
+        return this_call_cnt;
+}
 
-	if (!frame)
-		return -1;
 
-	local = frame->local;
+static uint64_t
+dht_bits_for (uint64_t num)
+{
+	uint64_t bits = 0, ctrl = 1;
 
-	LOCK (&frame->lock);
-	{
-		this_call_cnt = --local->call_cnt;
+	while (ctrl < num) {
+		ctrl *= 2;
+		bits ++;
 	}
-	UNLOCK (&frame->lock);
 
-	return this_call_cnt;
+	return bits;
 }
 
+/*
+ * A slightly "updated" version of the algorithm described in the commit log
+ * is used here.
+ *
+ * The only enhancement is that:
+ *
+ * - The number of bits used by the backend filesystem for HUGE d_off which
+ *   is described as 63, and
+ * - The number of bits used by the d_off presented by the transformation
+ *   upwards which is described as 64, are both made "configurable."
+ */
+
+
+#define BACKEND_D_OFF_BITS 63
+#define PRESENT_D_OFF_BITS 63
+
+#define ONE 1ULL
+#define MASK (~0ULL)
+#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
+#define BACKEND_MASK (MASK >> (64 - BACKEND_D_OFF_BITS))
+
+#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
+#define SHIFT_BITS (max (0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
 
 int
 dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
 {
-	dht_conf_t *conf = NULL;
-	int         cnt = 0;
-	int         max = 0;
-	uint64_t    y = 0;
+        dht_conf_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    y = 0;
+        uint64_t    hi_mask = 0;
+        uint64_t    off_mask = 0;
+        int         max_bits = 0;
+
+        if (x == ((uint64_t) -1)) {
+                y = (uint64_t) -1;
+                goto out;
+        }
 
+        conf = this->private;
+        if (!conf)
+                goto out;
 
-	if (x == ((uint64_t) -1)) {
-		y = (uint64_t) -1;
+        max = conf->subvolume_cnt;
+        cnt = dht_subvol_cnt (this, subvol);
+
+	if (max == 1) {
+		y = x;
 		goto out;
 	}
 
-	conf = this->private;
+        max_bits = dht_bits_for (max);
 
-	max = conf->subvolume_cnt;
-	cnt = dht_subvol_cnt (this, subvol);
+        hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
 
-	y = ((x * max) + cnt);
+        if (x & hi_mask) {
+                /* HUGE d_off */
+                off_mask = MASK << max_bits;
+                y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | cnt;
+        } else {
+                /* small d_off */
+                y = ((x * max) + cnt);
+        }
 
 out:
-	if (y_p)
-		*y_p = y;
+        if (y_p)
+                *y_p = y;
 
-	return 0;
+        return 0;
 }
 
+int
+dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
+                           xlator_t **subvol)
+{
+        char          *new_name  = NULL;
+        char          *new_path  = NULL;
+        xlator_list_t *trav      = NULL;
+        char           key[1024] = {0,};
+        int            ret       = 0; /* not found */
+
+        /* Why do other tasks if first required 'char' itself is not there */
+        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@'))
+                goto out;
+
+        trav = this->children;
+        while (trav) {
+                snprintf (key, 1024, "*@%s:%s", this->name, trav->xlator->name);
+                if (fnmatch (key, loc->name, FNM_NOESCAPE) == 0) {
+                        new_name = GF_CALLOC(strlen (loc->name),
+                                             sizeof (char),
+                                             gf_common_mt_char);
+                        if (!new_name)
+                                goto out;
+                        if (fnmatch (key, loc->path, FNM_NOESCAPE) == 0) {
+                                new_path = GF_CALLOC(strlen (loc->path),
+                                                     sizeof (char),
+                                                     gf_common_mt_char);
+                                if (!new_path)
+                                        goto out;
+                                strncpy (new_path, loc->path, (strlen (loc->path) -
+                                                               strlen (key) + 1));
+                        }
+                        strncpy (new_name, loc->name, (strlen (loc->name) -
+                                                       strlen (key) + 1));
+
+                        if (new_loc) {
+                                new_loc->path   = ((new_path) ? new_path:
+                                                   gf_strdup (loc->path));
+                                new_loc->name   = new_name;
+                                new_loc->inode  = inode_ref (loc->inode);
+                                new_loc->parent = inode_ref (loc->parent);
+                        }
+                        *subvol         = trav->xlator;
+                        ret = 1;  /* success */
+                        goto out;
+                }
+                trav = trav->next;
+        }
+out:
+        if (!ret) {
+                /* !success */
+                GF_FREE (new_path);
+                GF_FREE (new_name);
+        }
+        return ret;
+}
 
 int
 dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
-		  uint64_t *x_p)
+                  uint64_t *x_p)
 {
-	dht_conf_t *conf = NULL;
-	int         cnt = 0;
-	int         max = 0;
-	uint64_t    x = 0;
-	xlator_t   *subvol = 0;
+        dht_conf_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    x = 0;
+        xlator_t   *subvol = 0;
+        int         max_bits = 0;
+        uint64_t    off_mask = 0;
+        uint64_t    host_mask = 0;
+
+        if (!this->private)
+                return -1;
+
+        conf = this->private;
+        max = conf->subvolume_cnt;
+
+	if (max == 1) {
+		x = y;
+		cnt = 0;
+		goto out;
+	}
 
+        if (y & TOP_BIT) {
+                /* HUGE d_off */
+                max_bits = dht_bits_for (max);
+                off_mask = (MASK << max_bits);
+                host_mask = ~(off_mask);
 
-	conf = this->private;
-	max = conf->subvolume_cnt;
+                x = ((y & ~TOP_BIT) & off_mask) << SHIFT_BITS;
 
-	cnt = y % max;
-	x   = y / max;
+                cnt = y & host_mask;
+	} else {
+                /* small d_off */
+                cnt = y % max;
+                x = y / max;
+        }
 
-	subvol = conf->subvolumes[cnt];
+out:
+        subvol = conf->subvolumes[cnt];
 
-	if (subvol_p)
-		*subvol_p = subvol;
+        if (subvol_p)
+                *subvol_p = subvol;
 
-	if (x_p)
-		*x_p = x;
+        if (x_p)
+                *x_p = x;
 
-	return 0;
+        return 0;
 }
 
 
 void
-dht_local_wipe (dht_local_t *local)
+dht_local_wipe (xlator_t *this, dht_local_t *local)
 {
-	if (!local)
-		return;
+        if (!local)
+                return;
 
-	loc_wipe (&local->loc);
-	loc_wipe (&local->loc2);
+        loc_wipe (&local->loc);
+        loc_wipe (&local->loc2);
 
-	if (local->xattr)
-		dict_unref (local->xattr);
+        if (local->xattr)
+                dict_unref (local->xattr);
 
-	if (local->inode)
-		inode_unref (local->inode);
+        if (local->inode)
+                inode_unref (local->inode);
 
-	if (local->layout)
-		FREE (local->layout);
+        if (local->layout) {
+                dht_layout_unref (this, local->layout);
+                local->layout = NULL;
+        }
 
-	loc_wipe (&local->linkfile.loc);
+        loc_wipe (&local->linkfile.loc);
 
-	if (local->linkfile.xattr)
-		dict_unref (local->linkfile.xattr);
+        if (local->linkfile.xattr)
+                dict_unref (local->linkfile.xattr);
 
-	if (local->linkfile.inode)
-		inode_unref (local->linkfile.inode);
+        if (local->linkfile.inode)
+                inode_unref (local->linkfile.inode);
 
-	if (local->fd) {
-		fd_unref (local->fd);
-		local->fd = NULL;
-	}
-	
-	if (local->xattr_req)
-		dict_unref (local->xattr_req);
+        if (local->fd) {
+                fd_unref (local->fd);
+                local->fd = NULL;
+        }
+
+        if (local->params) {
+                dict_unref (local->params);
+                local->params = NULL;
+        }
+
+        if (local->xattr_req)
+                dict_unref (local->xattr_req);
+
+        if (local->selfheal.layout) {
+                dht_layout_unref (this, local->selfheal.layout);
+                local->selfheal.layout = NULL;
+        }
+
+        GF_FREE (local->newpath);
+
+        GF_FREE (local->key);
+
+        GF_FREE (local->rebalance.vector);
 
-	FREE (local);
+        if (local->rebalance.iobref)
+                iobref_unref (local->rebalance.iobref);
+
+        mem_put (local);
 }
 
 
 dht_local_t *
-dht_local_init (call_frame_t *frame)
+dht_local_init (call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
 {
-	dht_local_t *local = NULL;
+        dht_local_t *local = NULL;
+        inode_t     *inode = NULL;
+        int          ret   = 0;
+
+        local = mem_get0 (THIS->local_pool);
+        if (!local)
+                goto out;
+
+        if (loc) {
+                ret = loc_copy (&local->loc, loc);
+                if (ret)
+                        goto out;
 
-	/* TODO: use mem-pool */
-	local = CALLOC (1, sizeof (*local));
+                inode = loc->inode;
+        }
 
-	if (!local)
-		return NULL;
+        if (fd) {
+                local->fd = fd_ref (fd);
+                if (!inode)
+                        inode = fd->inode;
+        }
 
-	local->op_ret = -1;
-	local->op_errno = EUCLEAN;
+        local->op_ret   = -1;
+        local->op_errno = EUCLEAN;
+        local->fop      = fop;
 
-	frame->local = local;
+        if (inode) {
+                local->layout   = dht_layout_get (frame->this, inode);
+                local->cached_subvol = dht_subvol_get_cached (frame->this,
+                                                              inode);
+        }
 
-	return local;
-}
+        frame->local = local;
 
+out:
+        if (ret) {
+                if (local)
+                        mem_put (local);
+                local = NULL;
+        }
+        return local;
+}
 
-char *
-basestr (const char *str)
+xlator_t *
+dht_first_up_subvol (xlator_t *this)
 {
-        char *basestr = NULL;
+        dht_conf_t *conf = NULL;
+        xlator_t   *child = NULL;
+        int         i = 0;
+        time_t      time = 0;
 
-        basestr = strrchr (str, '/');
-        if (basestr)
-                basestr ++;
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        LOCK (&conf->subvolume_lock);
+        {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (conf->subvol_up_time[i]) {
+                                if (!time) {
+                                        time = conf->subvol_up_time[i];
+                                        child = conf->subvolumes[i];
+                                } else if (time > conf->subvol_up_time[i]) {
+                                        time  = conf->subvol_up_time[i];
+                                        child = conf->subvolumes[i];
+                                }
+                        }
+                }
+        }
+        UNLOCK (&conf->subvolume_lock);
 
-        return basestr;
+out:
+        return child;
 }
 
 xlator_t *
-dht_first_up_child (xlator_t *this)
+dht_last_up_subvol (xlator_t *this)
 {
-	dht_conf_t *conf = NULL;
-	xlator_t   *child = NULL;
-	int         i = 0;
-
-	conf = this->private;
-	
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (conf->subvolume_status[i]) {
-				child = conf->subvolumes[i];
-				break;
-			}
-		}
-	}
-	UNLOCK (&conf->subvolume_lock);
-	
-	return child;
+        dht_conf_t *conf = NULL;
+        xlator_t   *child = NULL;
+        int         i = 0;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        LOCK (&conf->subvolume_lock);
+        {
+                for (i = conf->subvolume_cnt-1; i >= 0; i--) {
+                        if (conf->subvolume_status[i]) {
+                                child = conf->subvolumes[i];
+                                break;
+                        }
+                }
+        }
+        UNLOCK (&conf->subvolume_lock);
+
+out:
+        return child;
 }
 
 xlator_t *
@@ -206,30 +426,40 @@ dht_subvol_get_hashed (xlator_t *this, loc_t *loc)
         dht_layout_t *layout = NULL;
         xlator_t     *subvol = NULL;
 
-        if (is_fs_root (loc)) {
-                subvol = dht_first_up_child (this);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, loc, out);
+
+        if (__is_root_gfid (loc->gfid)) {
+                subvol = dht_first_up_subvol (this);
                 goto out;
         }
 
+        GF_VALIDATE_OR_GOTO (this->name, loc->parent, out);
+        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
+
         layout = dht_layout_get (this, loc->parent);
 
         if (!layout) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "layout missing path=%s parent=%"PRId64,
-                        loc->path, loc->parent->ino);
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "layout missing path=%s parent=%s",
+                        loc->path, uuid_utoa (loc->parent->gfid));
                 goto out;
         }
 
         subvol = dht_layout_search (this, layout, loc->name);
 
         if (!subvol) {
-                gf_log (this->name, GF_LOG_ERROR,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "could not find subvolume for path=%s",
                         loc->path);
                 goto out;
         }
 
 out:
+        if (layout) {
+                dht_layout_unref (this, layout);
+        }
+
         return subvol;
 }
 
@@ -240,6 +470,8 @@ dht_subvol_get_cached (xlator_t *this, inode_t *inode)
         dht_layout_t *layout = NULL;
         xlator_t     *subvol = NULL;
 
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
         layout = dht_layout_get (this, inode);
 
@@ -247,9 +479,13 @@ dht_subvol_get_cached (xlator_t *this, inode_t *inode)
                 goto out;
         }
 
-	subvol = layout->list[0].xlator;
+        subvol = layout->list[0].xlator;
 
 out:
+        if (layout) {
+                dht_layout_unref (this, layout);
+        }
+
         return subvol;
 }
 
@@ -257,70 +493,655 @@ out:
 xlator_t *
 dht_subvol_next (xlator_t *this, xlator_t *prev)
 {
-	dht_conf_t *conf = NULL;
-	int         i = 0;
-	xlator_t   *next = NULL;
-
-	conf = this->private;
-
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (conf->subvolumes[i] == prev) {
-			if ((i + 1) < conf->subvolume_cnt)
-				next = conf->subvolumes[i + 1];
-			break;
-		}
-	}
+        dht_conf_t *conf = NULL;
+        int         i = 0;
+        xlator_t   *next = NULL;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == prev) {
+                        if ((i + 1) < conf->subvolume_cnt)
+                                next = conf->subvolumes[i + 1];
+                        break;
+                }
+        }
 
-	return next;
+out:
+        return next;
 }
 
+/* This func wraps around, if prev is actually the last subvol.
+ */
+xlator_t *
+dht_subvol_next_available (xlator_t *this, xlator_t *prev)
+{
+        dht_conf_t *conf = NULL;
+        int         i = 0;
+        xlator_t   *next = NULL;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
 
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == prev) {
+                        /* if prev is last in conf->subvolumes, then wrap
+                         * around.
+                         */
+                        if ((i + 1) < conf->subvolume_cnt) {
+                                next = conf->subvolumes[i + 1];
+                        } else {
+                                next = conf->subvolumes[0];
+                        }
+                        break;
+                }
+        }
+
+out:
+        return next;
+}
 int
 dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
 {
-	int i = 0;
-	int ret = -1;
-	dht_conf_t *conf = NULL;
+        int i = 0;
+        int ret = -1;
+        dht_conf_t *conf = NULL;
 
+        conf = this->private;
+        if (!conf)
+                goto out;
 
-	conf = this->private;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                        ret = i;
+                        break;
+                }
+        }
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (subvol == conf->subvolumes[i]) {
-			ret = i;
-			break;
-		}
-	}
+out:
+        return ret;
+}
+
+
+#define set_if_greater(a, b) do {               \
+                if ((a) < (b))                  \
+                        (a) = (b);              \
+        } while (0)
+
+
+#define set_if_greater_time(a, an, b, bn) do {                          \
+                if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))){    \
+                        (a) = (b);                                      \
+                        (an) = (bn);                                    \
+                }                                                       \
+        } while (0)                                                     \
+
+
+int
+dht_iatt_merge (xlator_t *this, struct iatt *to,
+                struct iatt *from, xlator_t *subvol)
+{
+        if (!from || !to)
+                return 0;
+
+        to->ia_dev      = from->ia_dev;
+
+        uuid_copy (to->ia_gfid, from->ia_gfid);
+
+        to->ia_ino      = from->ia_ino;
+        to->ia_prot     = from->ia_prot;
+        to->ia_type     = from->ia_type;
+        to->ia_nlink    = from->ia_nlink;
+        to->ia_rdev     = from->ia_rdev;
+        to->ia_size    += from->ia_size;
+        to->ia_blksize  = from->ia_blksize;
+        to->ia_blocks  += from->ia_blocks;
+
+        set_if_greater (to->ia_uid, from->ia_uid);
+        set_if_greater (to->ia_gid, from->ia_gid);
+
+        set_if_greater_time(to->ia_atime, to->ia_atime_nsec,
+                            from->ia_atime, from->ia_atime_nsec);
+        set_if_greater_time (to->ia_mtime, to->ia_mtime_nsec,
+                             from->ia_mtime, from->ia_mtime_nsec);
+        set_if_greater_time (to->ia_ctime, to->ia_ctime_nsec,
+                             from->ia_ctime, from->ia_ctime_nsec);
+
+        return 0;
+}
+
+int
+dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+{
+        if (!child) {
+                goto err;
+        }
+
+        if (strcmp (parent->path, "/") == 0)
+                gf_asprintf ((char **)&child->path, "/%s", name);
+        else
+                gf_asprintf ((char **)&child->path, "%s/%s", parent->path, name);
+
+        if (!child->path) {
+                goto err;
+        }
+
+        child->name = strrchr (child->path, '/');
+        if (child->name)
+                child->name++;
+
+        child->parent = inode_ref (parent->inode);
+        child->inode = inode_new (parent->inode->table);
+
+        if (!child->inode) {
+                goto err;
+        }
+
+        return 0;
+err:
+        loc_wipe (child);
+        return -1;
+}
+
+
+
+int
+dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
+{
+        xlator_list_t *subvols = NULL;
+        int            cnt = 0;
 
-	return ret;
+        if (!conf)
+                return -1;
+
+        for (subvols = this->children; subvols; subvols = subvols->next)
+                cnt++;
+
+        conf->subvolumes = GF_CALLOC (cnt, sizeof (xlator_t *),
+                                      gf_dht_mt_xlator_t);
+        if (!conf->subvolumes) {
+                return -1;
+        }
+        conf->subvolume_cnt = cnt;
+
+        cnt = 0;
+        for (subvols = this->children; subvols; subvols = subvols->next)
+                conf->subvolumes[cnt++] = subvols->xlator;
+
+        conf->subvolume_status = GF_CALLOC (cnt, sizeof (char),
+                                            gf_dht_mt_char);
+        if (!conf->subvolume_status) {
+                return -1;
+        }
+
+        conf->last_event = GF_CALLOC (cnt, sizeof (int),
+                                      gf_dht_mt_char);
+        if (!conf->last_event) {
+                return -1;
+        }
+
+        conf->subvol_up_time = GF_CALLOC (cnt, sizeof (time_t),
+                                          gf_dht_mt_subvol_time);
+        if (!conf->subvol_up_time) {
+                return -1;
+        }
+
+        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
+                                    gf_dht_mt_dht_du_t);
+        if (!conf->du_stats) {
+                return -1;
+        }
+
+        conf->decommissioned_bricks = GF_CALLOC (cnt, sizeof (xlator_t *),
+                                                 gf_dht_mt_xlator_t);
+        if (!conf->decommissioned_bricks) {
+                return -1;
+        }
+
+        return 0;
+}
+
+
+
+
+static int
+dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data)
+{
+        dht_local_t *local = NULL;
+
+        local = frame->local;
+
+        local->rebalance.target_op_fn (THIS, frame, op_ret);
+
+        return 0;
+}
+
+
+int
+dht_migration_complete_check_task (void *data)
+{
+        int           ret      = -1;
+        xlator_t     *src_node = NULL;
+        xlator_t     *dst_node = NULL;
+        dht_local_t  *local    = NULL;
+        dict_t       *dict     = NULL;
+        dht_layout_t *layout   = NULL;
+        struct iatt   stbuf    = {0,};
+        xlator_t     *this     = NULL;
+        call_frame_t *frame    = NULL;
+        loc_t         tmp_loc  = {0,};
+        char         *path     = NULL;
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        uint64_t      tmp_subvol = 0;
+        int           open_failed = 0;
+
+        this  = THIS;
+        frame = data;
+        local = frame->local;
+        conf = this->private;
+
+        src_node = local->cached_subvol;
+
+        if (!local->loc.inode && !local->fd)
+                goto out;
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+
+        if (!local->loc.inode) {
+                ret = syncop_fgetxattr (src_node, local->fd, &dict,
+                                        conf->link_xattr_name);
+        } else {
+                SYNCTASK_SETID (0, 0);
+                ret = syncop_getxattr (src_node, &local->loc, &dict,
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        }
+
+        if (!ret)
+                dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
+
+        if (ret) {
+                if ((errno != ENOENT) || (!local->loc.inode)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to get the 'linkto' xattr %s",
+                                local->loc.path, strerror (errno));
+                        goto out;
+                }
+                /* Need to do lookup on hashed subvol, then get the file */
+                ret = syncop_lookup (this, &local->loc, NULL, &stbuf, NULL,
+                                     NULL);
+                if (ret)
+                        goto out;
+                dst_node = dht_subvol_get_cached (this, local->loc.inode);
+        }
+
+        if (!dst_node) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to get the destination node",
+                        local->loc.path);
+                ret = -1;
+                goto out;
+        }
+
+        /* lookup on dst */
+        if (local->loc.inode) {
+                ret = syncop_lookup (dst_node, &local->loc, NULL, &stbuf, NULL, NULL);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to lookup the file on %s",
+                                local->loc.path, dst_node->name);
+                        goto out;
+                }
+
+                if (uuid_compare (stbuf.ia_gfid, local->loc.inode->gfid)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: gfid different on the target file on %s",
+                                local->loc.path, dst_node->name);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* update inode ctx (the layout) */
+        dht_layout_unref (this, local->layout);
+
+        ret = dht_layout_preset (this, dst_node, inode);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: could not set preset layout for subvol %s",
+                        local->loc.path, dst_node->name);
+                ret   = -1;
+                goto out;
+        }
+
+        layout = dht_layout_for_subvol (this, dst_node);
+        if (!layout) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: no pre-set layout for subvolume %s",
+                        local->loc.path, dst_node ? dst_node->name : "<nil>");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dht_layout_set (this, inode, layout);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set the new layout",
+                        local->loc.path);
+                goto out;
+        }
+
+        local->cached_subvol = dst_node;
+        ret = 0;
+
+        /* once we detect the migration complete, the inode-ctx2 is no more
+           required.. delete the ctx and also, it means, open() already
+           done on all the fd of inode */
+        ret = inode_ctx_reset1 (inode, this, &tmp_subvol);
+        if (tmp_subvol)
+                goto out;
+
+        if (list_empty (&inode->fd_list))
+                goto out;
+
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID(0, 0);
+
+        /* perform 'open()' on all the fd's present on the inode */
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                ret = syncop_open (dst_node, &tmp_loc,
+                                   iter_fd->flags, iter_fd);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                }
+        }
+        GF_FREE (path);
+
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        if (open_failed) {
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+
+        return ret;
+}
+
+int
+dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
+{
+        int         ret     = -1;
+
+        ret = synctask_new (this->ctx->env, dht_migration_complete_check_task,
+                            dht_migration_complete_check_done,
+                            frame, frame);
+        return ret;
 }
 
+/* During 'in-progress' state, both nodes should have the file */
+static int
+dht_inprogress_check_done (int op_ret, call_frame_t *sync_frame, void *data)
+{
+        dht_local_t *local = NULL;
+
+        local = sync_frame->local;
+
+        local->rebalance.target_op_fn (THIS, sync_frame, op_ret);
+
+        return 0;
+}
+
+static int
+dht_rebalance_inprogress_task (void *data)
+{
+        int           ret      = -1;
+        xlator_t     *src_node = NULL;
+        xlator_t     *dst_node = NULL;
+        dht_local_t  *local    = NULL;
+        dict_t       *dict     = NULL;
+        call_frame_t *frame    = NULL;
+        xlator_t     *this     = NULL;
+        char         *path     = NULL;
+        struct iatt   stbuf    = {0,};
+        loc_t         tmp_loc  = {0,};
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        int           open_failed = 0;
+
+        this  = THIS;
+        frame = data;
+        local = frame->local;
+        conf = this->private;
+
+        src_node = local->cached_subvol;
+
+        if (!local->loc.inode && !local->fd)
+                goto out;
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+        if (local->loc.inode) {
+                SYNCTASK_SETID (0, 0);
+                ret = syncop_getxattr (src_node, &local->loc, &dict,
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        } else {
+                ret = syncop_fgetxattr (src_node, local->fd, &dict,
+                                        conf->link_xattr_name);
+        }
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to get the 'linkto' xattr %s",
+                        local->loc.path, strerror (errno));
+                        goto out;
+        }
+
+        dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
+        if (!dst_node) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to get the 'linkto' xattr from dict",
+                        local->loc.path);
+                ret = -1;
+                goto out;
+        }
 
-#define set_if_greater(a, b) do {		\
-		if ((a) < (b))			\
-			(a) = (b);		\
-	} while (0)
+        local->rebalance.target_node = dst_node;
+
+        if (local->loc.inode) {
+                /* lookup on dst */
+                ret = syncop_lookup (dst_node, &local->loc, NULL,
+                                     &stbuf, NULL, NULL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to lookup the file on %s",
+                                local->loc.path, dst_node->name);
+                        goto out;
+                }
+
+                if (uuid_compare (stbuf.ia_gfid, local->loc.inode->gfid)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: gfid different on the target file on %s",
+                                local->loc.path, dst_node->name);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+
+        if (list_empty (&inode->fd_list))
+                goto done;
+
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID (0, 0);
+
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                ret = syncop_open (dst_node, &tmp_loc,
+                                   iter_fd->flags, iter_fd);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to send open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                }
+        }
+        GF_FREE (path);
+
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
+                goto out;
+        }
+
+done:
+        ret = dht_inode_ctx_set1 (this, inode, dst_node);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set inode-ctx target file at %s",
+                        local->loc.path, dst_node->name);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
 
 int
-dht_stat_merge (xlator_t *this, struct stat *to,
-		struct stat *from, xlator_t *subvol)
+dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
 {
-	to->st_dev      = from->st_dev;
 
-	dht_itransform (this, subvol, from->st_ino, &to->st_ino);
+        int         ret     = -1;
+
+        ret = synctask_new (this->ctx->env, dht_rebalance_inprogress_task,
+                            dht_inprogress_check_done,
+                            frame, frame);
+        return ret;
+}
+
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t *layout_int)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+        if (!ret && ctx) {
+                ctx->layout = layout_int;
+        } else {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return ret;
+                ctx->layout = layout_int;
+        }
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+
+        return ret;
+}
+
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t post)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        dht_stat_time_t         *time           = 0;
+        int                      ret            = -1;
+
+        GF_VALIDATE_OR_GOTO (this->name, stat, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (ret) {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return -1;
+        }
 
-	to->st_mode     = from->st_mode;
-	to->st_nlink    = from->st_nlink;
-	to->st_uid      = from->st_uid;
-	to->st_gid      = from->st_gid;
-	to->st_rdev     = from->st_rdev;
-	to->st_size    += from->st_size;
-	to->st_blksize  = from->st_blksize;
-	to->st_blocks  += from->st_blocks;
+        time = &ctx->time;
 
-	set_if_greater (to->st_atime, from->st_atime);
-	set_if_greater (to->st_mtime, from->st_mtime);
-	set_if_greater (to->st_ctime, from->st_ctime);
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec,
+                        stat->ia_mtime, stat->ia_mtime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec,
+                        stat->ia_ctime, stat->ia_ctime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec,
+                        stat->ia_atime, stat->ia_atime_nsec, inode, post);
 
-	return 0;
+        ret = dht_inode_ctx_set (inode, this, ctx);
+out:
+        return 0;
+}
+
+int
+dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+{
+        int             ret     = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+
+        if (ret)
+                return ret;
+
+        if (ctx)
+                *ctx = (dht_inode_ctx_t *) ctx_int;
+out:
+        return ret;
+}
+
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
+{
+        int             ret = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+
+        ctx_int = (long)ctx;
+        ret = inode_ctx_set (inode, this, &ctx_int);
+out:
+        return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
new file mode 100644
index 000000000..ece84151a
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -0,0 +1,1139 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "dht-common.h"
+
+int dht_access2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_readv2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_attr2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_open2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_flush2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_lk2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_fsync2 (xlator_t *this, call_frame_t *frame, int ret);
+
+int
+dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = 0;
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (!op_ret || (local->call_cnt != 1))
+                goto out;
+
+        /* rebalance would have happened */
+        local->rebalance.target_op_fn = dht_open2;
+        ret = dht_rebalance_complete_check (this, frame);
+        if (!ret)
+                return 0;
+
+out:
+        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd, xdata);
+
+        return 0;
+}
+
+int
+dht_open2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t *local  = NULL;
+        xlator_t    *subvol = NULL;
+        int          op_errno = EINVAL;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        op_errno = ENOENT;
+        if (op_ret)
+                goto out;
+
+        local->call_cnt = 2;
+        subvol = local->cached_subvol;
+
+        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
+                    &local->loc, local->rebalance.flags, local->fd,
+                    NULL);
+        return 0;
+
+out:
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+dht_open (call_frame_t *frame, xlator_t *this,
+          loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, loc, fd, GF_FOP_OPEN);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->rebalance.flags = flags;
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
+                    loc, flags, fd, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+{
+        xlator_t     *subvol = 0;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        inode_t      *inode = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                local->op_errno = op_errno;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        local->op_errno = op_errno;
+        /* Check if the rebalance phase2 is true */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
+                        /* Phase 2 of migration */
+                        local->rebalance.target_op_fn = dht_attr2;
+                        ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
+                } else {
+                        /* value is already set in fd_ctx, that means no need
+                           to check for whether its complete or not. */
+                        dht_attr2 (this, frame, 0);
+                        return 0;
+                }
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_attr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t *local  = NULL;
+        xlator_t    *subvol = NULL;
+        int          op_errno = EINVAL;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        op_errno = local->op_errno;
+        if (op_ret == -1)
+                goto out;
+
+        subvol = local->cached_subvol;
+        local->call_cnt = 2;
+
+        if (local->fop == GF_FOP_FSTAT) {
+                STACK_WIND (frame, dht_file_attr_cbk, subvol,
+                            subvol->fops->fstat, local->fd, NULL);
+        } else {
+                STACK_WIND (frame, dht_file_attr_cbk, subvol,
+                            subvol->fops->stat, &local->loc, NULL);
+        }
+        return 0;
+
+out:
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+
+                        goto unlock;
+                }
+
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
+out:
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
+                                  &local->stbuf, xdata);
+        }
+err:
+        return 0;
+}
+
+int
+dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_STAT);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (IA_ISREG (loc->inode->ia_type)) {
+                local->call_cnt = 1;
+
+                subvol = local->cached_subvol;
+
+                STACK_WIND (frame, dht_file_attr_cbk, subvol,
+                            subvol->fops->stat, loc, xdata);
+
+                return 0;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                subvol = layout->list[i].xlator;
+
+                STACK_WIND (frame, dht_attr_cbk,
+                            subvol, subvol->fops->stat,
+                            loc, xdata);
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FSTAT);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (IA_ISREG (fd->inode->ia_type)) {
+                local->call_cnt = 1;
+
+                subvol = local->cached_subvol;
+
+                STACK_WIND (frame, dht_file_attr_cbk, subvol,
+                            subvol->fops->fstat, fd, xdata);
+
+                return 0;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                subvol = layout->list[i].xlator;
+                STACK_WIND (frame, dht_attr_cbk,
+                            subvol, subvol->fops->fstat,
+                            fd, xdata);
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno,
+               struct iovec *vector, int count, struct iatt *stbuf,
+               struct iobref *iobref, dict_t *xdata)
+{
+        dht_local_t *local      = NULL;
+        int          ret        = 0;
+        inode_t     *inode      = NULL;
+        xlator_t    *subvol = 0;
+
+        local = frame->local;
+        if (!local) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        /* This is already second try, no need for re-check */
+        if (local->call_cnt != 1)
+                goto out;
+
+        if ((op_ret == -1) && (op_errno != ENOENT))
+                goto out;
+
+        local->op_errno = op_errno;
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
+                /* File would be migrated to other node */
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
+                        local->rebalance.target_op_fn = dht_readv2;
+                        ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
+                } else {
+                        /* value is already set in fd_ctx, that means no need
+                           to check for whether its complete or not. */
+                        dht_readv2 (this, frame, 0);
+                        return 0;
+                }
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
+                          iobref, xdata);
+
+        return 0;
+}
+
+int
+dht_readv2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t *local  = NULL;
+        xlator_t    *subvol = NULL;
+        int          op_errno = EINVAL;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        op_errno = local->op_errno;
+        if (op_ret == -1)
+                goto out;
+
+        local->call_cnt = 2;
+        subvol = local->cached_subvol;
+
+        STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
+                    local->fd, local->rebalance.size, local->rebalance.offset,
+                    local->rebalance.flags, NULL);
+
+        return 0;
+
+out:
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+dht_readv (call_frame_t *frame, xlator_t *this,
+           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_READ);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->rebalance.offset = off;
+        local->rebalance.size   = size;
+        local->rebalance.flags  = flags;
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_readv_cbk,
+                    subvol, subvol->fops->readv,
+                    fd, size, off, flags, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, dict_t *xdata)
+{
+        int          ret = -1;
+        dht_local_t *local = NULL;
+        xlator_t    *subvol = NULL;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev = cookie;
+
+        if (!prev || !prev->this)
+                goto out;
+        if (local->call_cnt != 1)
+                goto out;
+        if ((op_ret == -1) && (op_errno == ENOTCONN) &&
+            IA_ISDIR(local->loc.inode->ia_type)) {
+
+                subvol = dht_subvol_next_available (this, prev->this);
+                if (!subvol)
+                        goto out;
+
+                /* check if we are done with visiting every node */
+                if (subvol == local->cached_subvol) {
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                            &local->loc, local->rebalance.flags, NULL);
+                return 0;
+        }
+        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                /* File would be migrated to other node */
+                local->op_errno = op_errno;
+                local->rebalance.target_op_fn = dht_access2;
+                ret = dht_rebalance_complete_check (frame->this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+dht_access2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t *local  = NULL;
+        xlator_t    *subvol = NULL;
+        int          op_errno = EINVAL;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        op_errno = local->op_errno;
+        if (op_ret == -1)
+                goto out;
+
+        local->call_cnt = 2;
+        subvol = local->cached_subvol;
+
+        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                    &local->loc, local->rebalance.flags, NULL);
+
+        return 0;
+
+out:
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+        return 0;
+}
+
+
+int
+dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_ACCESS);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.flags = mask;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                    loc, mask, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno, dict_t *xdata)
+{
+        dht_local_t  *local  = NULL;
+        inode_t      *inode  = NULL;
+        xlator_t     *subvol = 0;
+
+        local = frame->local;
+
+        local->op_errno = op_errno;
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        /* If context is set, then send flush() it to the destination */
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (subvol) {
+                dht_flush2 (this, frame, 0);
+                return 0;
+        }
+
+out:
+        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int
+dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND (frame, dht_flush_cbk,
+                    subvol, subvol->fops->flush, local->fd, NULL);
+
+        return 0;
+}
+
+
+int
+dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FLUSH);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_flush_cbk,
+                    subvol, subvol->fops->flush, fd, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        inode_t      *inode = NULL;
+        xlator_t     *subvol = 0;
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+        if (op_ret == -1 && (op_errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+
+        local->op_errno = op_errno;
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (!subvol) {
+                local->rebalance.target_op_fn = dht_fsync2;
+
+                /* Check if the rebalance phase1 is true */
+                if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                        dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                        dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+
+                        ret = dht_rebalance_in_progress_check (this, frame);
+                }
+
+                /* Check if the rebalance phase2 is true */
+                if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                        ret = dht_rebalance_complete_check (this, frame);
+                }
+                if (!ret)
+                        return 0;
+        } else {
+                dht_fsync2 (this, frame, 0);
+                return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+
+        return 0;
+}
+
+int
+dht_fsync2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
+                    local->fd, local->rebalance.flags, NULL);
+
+        return 0;
+}
+
+int
+dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+           dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FSYNC);
+        if (!local) {
+                op_errno = ENOMEM;
+
+                goto err;
+        }
+
+        local->call_cnt = 1;
+        local->rebalance.flags = datasync;
+
+        subvol = local->cached_subvol;
+
+        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
+                    fd, datasync, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+/* TODO: for 'lk()' call, we need some other special error, may be ESTALE to
+   indicate that lock migration happened on the fd, so we can consider it as
+   phase 2 of migration */
+int
+dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+            int op_ret, int op_errno, struct gf_flock *flock, dict_t *xdata)
+{
+        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock, xdata);
+
+        return 0;
+}
+
+
+int
+dht_lk (call_frame_t *frame, xlator_t *this,
+        fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        subvol = dht_subvol_get_cached (this, fd->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        /* TODO: for rebalance, we need to preserve the fop arguments */
+        STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, fd,
+                    cmd, flock, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+/* Symlinks are currently not migrated, so no need for any check here */
+int
+dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, const char *path,
+                  struct iatt *stbuf, dict_t *xdata)
+{
+        dht_local_t *local = NULL;
+
+        local = frame->local;
+        if (op_ret == -1)
+                goto err;
+
+        if (!local) {
+                op_ret = -1;
+                op_errno = EINVAL;
+        }
+
+err:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf, xdata);
+
+        return 0;
+}
+
+
+int
+dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+              dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_READLINK);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_readlink_cbk,
+                    subvol, subvol->fops->readlink,
+                    loc, size, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+/* Currently no translators on top of 'distribute' will be using
+ * below fops, hence not implementing 'migration' related checks
+ */
+
+int
+dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_XATTROP);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = 1;
+
+        STACK_WIND (frame,
+                    dht_xattrop_cbk,
+                    subvol, subvol->fops->xattrop,
+                    loc, flags, dict, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+dht_fxattrop (call_frame_t *frame, xlator_t *this,
+              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        subvol = dht_subvol_get_cached (this, fd->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame,
+                    dht_fxattrop_cbk,
+                    subvol, subvol->fops->fxattrop,
+                    fd, flags, dict, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_inodelk_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+
+{
+        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int32_t
+dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+             loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_INODELK);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = 1;
+
+        STACK_WIND (frame,
+                    dht_inodelk_cbk,
+                    subvol, subvol->fops->inodelk,
+                    volume, loc, cmd, lock, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+dht_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+
+{
+        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        subvol = dht_subvol_get_cached (this, fd->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+
+        STACK_WIND (frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+
+        return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
new file mode 100644
index 000000000..4b3f3a049
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -0,0 +1,1013 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "dht-common.h"
+
+int dht_writev2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_truncate2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_setattr2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret);
+
+int
+dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t *local = NULL;
+        int          ret   = -1;
+        xlator_t    *subvol = NULL;
+
+        if (op_ret == -1 && (op_errno != ENOENT)) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (!local) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                /* preserve the modes of source */
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+
+        local->rebalance.target_op_fn = dht_writev2;
+
+        local->op_errno = op_errno;
+        /* Phase 2 of migration */
+        if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+
+                ret = dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_writev2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+
+        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+int
+dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND (frame, dht_writev_cbk,
+                    subvol, subvol->fops->writev,
+                    local->fd, local->rebalance.vector, local->rebalance.count,
+                    local->rebalance.offset, local->rebalance.flags,
+                    local->rebalance.iobref, NULL);
+
+        return 0;
+}
+
+int
+dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+            struct iovec *vector, int count, off_t off, uint32_t flags,
+            struct iobref *iobref, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_WRITE);
+        if (!local) {
+
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+
+        local->rebalance.vector = iov_dup (vector, count);
+        local->rebalance.offset = off;
+        local->rebalance.count = count;
+        local->rebalance.flags = flags;
+        local->rebalance.iobref = iobref_ref (iobref);
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_writev_cbk,
+                    subvol, subvol->fops->writev,
+                    fd, vector, count, off, flags, iobref, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
+int
+dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+        inode_t      *inode = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+
+        local->rebalance.target_op_fn = dht_truncate2;
+
+        local->op_errno = op_errno;
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                dht_inode_ctx_get1 (this, inode, &subvol);
+                if (subvol) {
+                        dht_truncate2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+
+int
+dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+        inode_t      *inode = NULL;
+
+        local = frame->local;
+
+        inode = local->fd ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        if (local->fop == GF_FOP_TRUNCATE) {
+                STACK_WIND (frame, dht_truncate_cbk, subvol,
+                            subvol->fops->truncate, &local->loc,
+                            local->rebalance.offset, NULL);
+        } else {
+                STACK_WIND (frame, dht_truncate_cbk, subvol,
+                            subvol->fops->ftruncate, local->fd,
+                            local->rebalance.offset, NULL);
+        }
+
+        return 0;
+}
+
+int
+dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_TRUNCATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_truncate_cbk,
+                    subvol, subvol->fops->truncate,
+                    loc, offset, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FTRUNCATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_truncate_cbk,
+                    subvol, subvol->fops->ftruncate,
+                    fd, offset, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_fallocate2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_fallocate2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_fallocate_cbk, subvol, subvol->fops->fallocate,
+		   local->fd, local->rebalance.flags, local->rebalance.offset,
+		   local->rebalance.size, NULL);
+
+        return 0;
+}
+
+int
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+	      off_t offset, size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FALLOCATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+	local->rebalance.flags = mode;
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_fallocate_cbk,
+                    subvol, subvol->fops->fallocate,
+                    fd, mode, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_discard2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_discard2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (discard, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_discard_cbk, subvol, subvol->fops->discard,
+		   local->fd, local->rebalance.offset, local->rebalance.size,
+		   NULL);
+
+        return 0;
+}
+
+int
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	    size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_DISCARD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_discard_cbk, subvol, subvol->fops->discard,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev  = NULL;
+        int           ret   = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_zerofill2;
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                ret = fd_ctx_get (local->fd, this, NULL);
+                if (!ret) {
+                        dht_zerofill2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local          = NULL;
+        xlator_t     *subvol         = NULL;
+        uint64_t      tmp_subvol     = 0;
+        int           ret            = -1;
+
+        local = frame->local;
+
+        if (local->fd)
+                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
+        if (!ret)
+                subvol = (xlator_t *)(long)tmp_subvol;
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND(frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                   local->fd, local->rebalance.offset, local->rebalance.size,
+                   NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol       = NULL;
+        int           op_errno     = -1;
+        dht_local_t  *local        = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_ZEROFILL);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
+/* handle cases of migration here for 'setattr()' calls */
+int
+dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+        if ((op_ret == -1) && (op_errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        local->rebalance.target_op_fn = dht_setattr2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* At the end of the migration process, whatever 'attr' we
+           have on source file will be migrated to destination file
+           in one shot, hence we don't need to check for in progress
+           state here (ie, PHASE1) */
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+
+        return 0;
+}
+
+int
+dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+        inode_t      *inode = NULL;
+
+        local = frame->local;
+
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        if (local->fop == GF_FOP_SETATTR) {
+                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
+                            subvol->fops->setattr, &local->loc,
+                            &local->rebalance.stbuf, local->rebalance.flags,
+                            NULL);
+        } else {
+                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
+                            subvol->fops->fsetattr, local->fd,
+                            &local->rebalance.stbuf, local->rebalance.flags,
+                            NULL);
+        }
+
+        return 0;
+}
+
+
+/* Keep the existing code same for all the cases other than regular file */
+int
+dht_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int op_ret, int op_errno, struct iatt *statpre,
+                 struct iatt *statpost, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+
+
+        local = frame->local;
+        prev = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
+
+                dht_iatt_merge (this, &local->prebuf, statpre, prev->this);
+                dht_iatt_merge (this, &local->stbuf, statpost, prev->this);
+
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt))
+                DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
+                                  &local->prebuf, &local->stbuf, xdata);
+
+        return 0;
+}
+
+
+int
+dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        dht_layout_t *layout = NULL;
+        dht_local_t  *local  = NULL;
+        int           op_errno = -1;
+        int           i = -1;
+        int           call_cnt = 0;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_SETATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (!layout_is_sane (layout)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "layout is not sane for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (IA_ISREG (loc->inode->ia_type)) {
+                /* in the regular file _cbk(), we need to check for
+                   migration possibilities */
+                local->rebalance.stbuf = *stbuf;
+                local->rebalance.flags = valid;
+                local->call_cnt = 1;
+                subvol = local->cached_subvol;
+
+                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
+                            subvol->fops->setattr,
+                            loc, stbuf, valid, xdata);
+
+                return 0;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_setattr_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->setattr,
+                            loc, stbuf, valid, xdata);
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+              int32_t valid, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        dht_layout_t *layout = NULL;
+        dht_local_t  *local  = NULL;
+        int           op_errno = -1;
+        int           i = -1;
+        int           call_cnt = 0;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (!layout_is_sane (layout)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "layout is not sane for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (IA_ISREG (fd->inode->ia_type)) {
+                /* in the regular file _cbk(), we need to check for
+                   migration possibilities */
+                local->rebalance.stbuf = *stbuf;
+                local->rebalance.flags = valid;
+                local->call_cnt = 1;
+                subvol = local->cached_subvol;
+
+                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
+                            subvol->fops->fsetattr,
+                            fd, stbuf, valid, xdata);
+
+                return 0;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_setattr_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->fsetattr,
+                            fd, stbuf, valid, xdata);
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 25cfef39a..38e9970a7 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -38,545 +29,764 @@
 dht_layout_t *
 dht_layout_new (xlator_t *this, int cnt)
 {
-	dht_layout_t *layout = NULL;
+        dht_layout_t *layout = NULL;
+        dht_conf_t   *conf = NULL;
 
 
-	layout = CALLOC (1, layout_size (cnt));
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto out;
-	}
+        conf = this->private;
+
+        layout = GF_CALLOC (1, layout_size (cnt),
+                            gf_dht_mt_dht_layout_t);
+        if (!layout) {
+                goto out;
+        }
+
+        layout->type = DHT_HASH_TYPE_DM;
+        layout->cnt = cnt;
 
-	layout->cnt = cnt;
+        if (conf) {
+                layout->spread_cnt = conf->dir_spread_cnt;
+                layout->gen = conf->gen;
+        }
 
+        layout->ref = 1;
 out:
-	return layout;
+        return layout;
 }
 
 
 dht_layout_t *
 dht_layout_get (xlator_t *this, inode_t *inode)
 {
-        uint64_t layout = 0;
-        int      ret    = -1;
+        dht_conf_t   *conf = NULL;
+        dht_layout_t *layout = NULL;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        LOCK (&conf->layout_lock);
+        {
+                dht_inode_ctx_layout_get (inode, this, &layout);
+                if (layout) {
+                        layout->ref++;
+                }
+        }
+        UNLOCK (&conf->layout_lock);
+
+out:
+        return layout;
+}
 
-        ret = inode_ctx_get (inode, this, &layout);
 
-        return (dht_layout_t *)(long)layout;
+int
+dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
+{
+        dht_conf_t   *conf = NULL;
+        int           oldret = -1;
+        int           ret = 0;
+        dht_layout_t *old_layout;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        LOCK (&conf->layout_lock);
+        {
+                oldret = dht_inode_ctx_layout_get (inode, this, &old_layout);
+                layout->ref++;
+                dht_inode_ctx_layout_set (inode, this, layout);
+        }
+        UNLOCK (&conf->layout_lock);
+
+        if (!oldret) {
+                dht_layout_unref (this, old_layout);
+        }
+
+out:
+        return ret;
 }
 
 
-xlator_t *
-dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
+void
+dht_layout_unref (xlator_t *this, dht_layout_t *layout)
 {
-	uint32_t   hash = 0;
-        xlator_t  *subvol = NULL;
-	int        i = 0;
-	int        ret = 0;
+        dht_conf_t  *conf = NULL;
+        int          ref = 0;
 
+        if (!layout || layout->preset || !this->private)
+                return;
 
-	ret = dht_hash_compute (layout->type, name, &hash);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"hash computation failed for type=%d name=%s",
-			layout->type, name);
-		goto out;
-	}
+        conf = this->private;
 
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].start <= hash
-		    && layout->list[i].stop >= hash) {
-			subvol = layout->list[i].xlator;
-			break;
-		}
-	}
+        LOCK (&conf->layout_lock);
+        {
+                ref = --layout->ref;
+        }
+        UNLOCK (&conf->layout_lock);
 
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume for hash (value) = %u", hash);
-	}
+        if (!ref)
+                GF_FREE (layout);
+}
+
+
+dht_layout_t *
+dht_layout_ref (xlator_t *this, dht_layout_t *layout)
+{
+        dht_conf_t  *conf = NULL;
+
+        if (layout->preset || !this->private)
+                return layout;
+
+        conf = this->private;
+        LOCK (&conf->layout_lock);
+        {
+                layout->ref++;
+        }
+        UNLOCK (&conf->layout_lock);
+
+        return layout;
+}
+
+
+xlator_t *
+dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
+{
+        uint32_t   hash = 0;
+        xlator_t  *subvol = NULL;
+        int        i = 0;
+        int        ret = 0;
+
+
+        ret = dht_hash_compute (this, layout->type, name, &hash);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "hash computation failed for type=%d name=%s",
+                        layout->type, name);
+                goto out;
+        }
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].start <= hash
+                    && layout->list[i].stop >= hash) {
+                        subvol = layout->list[i].xlator;
+                        break;
+                }
+        }
+
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "no subvolume for hash (value) = %u", hash);
+        }
 
 out:
-	return subvol;
+        return subvol;
 }
 
 
 dht_layout_t *
 dht_layout_for_subvol (xlator_t *this, xlator_t *subvol)
 {
-	dht_conf_t   *conf = NULL;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-
+        dht_conf_t   *conf = NULL;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
 
-	conf = this->private;
+        conf = this->private;
+        if (!conf)
+                goto out;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (conf->subvolumes[i] == subvol) {
-			layout = conf->file_layouts[i];
-			break;
-		}
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == subvol) {
+                        layout = conf->file_layouts[i];
+                        break;
+                }
+        }
 
-	return layout;
+out:
+        return layout;
 }
 
 
 int
 dht_layouts_init (xlator_t *this, dht_conf_t *conf)
 {
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-	int           ret = -1;
-	
-
-	conf->file_layouts = CALLOC (conf->subvolume_cnt,
-				     sizeof (dht_layout_t *));
-	if (!conf->file_layouts) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto out;
-	}
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           ret = -1;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		layout = dht_layout_new (this, 1);
+        if (!conf)
+                goto out;
 
-		if (!layout) {
-			goto out;
-		}
+        conf->file_layouts = GF_CALLOC (conf->subvolume_cnt,
+                                        sizeof (dht_layout_t *),
+                                        gf_dht_mt_dht_layout_t);
+        if (!conf->file_layouts) {
+                goto out;
+        }
 
-		layout->preset = 1;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                layout = dht_layout_new (this, 1);
 
-		layout->list[0].xlator = conf->subvolumes[i];
+                if (!layout) {
+                        goto out;
+                }
 
-		conf->file_layouts[i] = layout;
-	}
+                layout->preset = 1;
 
-	ret = 0;
+                layout->list[0].xlator = conf->subvolumes[i];
+
+                conf->file_layouts[i] = layout;
+        }
+
+        ret = 0;
 out:
-	return ret;
+        return ret;
 }
 
 
 int
 dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-			 int pos, int32_t **disk_layout_p)
+                         int pos, int32_t **disk_layout_p)
 {
-	int      ret = -1;
-	int32_t *disk_layout = NULL;
-
-	disk_layout = CALLOC (5, sizeof (int));
-	if (!disk_layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto out;
-	}
+        int      ret = -1;
+        int32_t *disk_layout = NULL;
 
-	disk_layout[0] = hton32 (1);
-	disk_layout[1] = hton32 (layout->type);
-	disk_layout[2] = hton32 (layout->list[pos].start);
-	disk_layout[3] = hton32 (layout->list[pos].stop);
+        disk_layout = GF_CALLOC (5, sizeof (int),
+                                 gf_dht_mt_int32_t);
+        if (!disk_layout) {
+                goto out;
+        }
 
-	if (disk_layout_p)
-		*disk_layout_p = disk_layout;
-	ret = 0;
+        disk_layout[0] = hton32 (1);
+        disk_layout[1] = hton32 (layout->type);
+        disk_layout[2] = hton32 (layout->list[pos].start);
+        disk_layout[3] = hton32 (layout->list[pos].stop);
+
+        if (disk_layout_p)
+                *disk_layout_p = disk_layout;
+        else
+                GF_FREE (disk_layout);
+
+        ret = 0;
 
 out:
-	return ret;
+        return ret;
 }
 
 
 int
 dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
-		       int pos, int32_t *disk_layout)
+		       int pos, void *disk_layout_raw, int disk_layout_len)
 {
-	int      cnt = 0;
-	int      type = 0;
-	int      start_off = 0;
-	int      stop_off = 0;
-
-	/* TODO: assert disk_layout_ptr is of required length */
+        int      cnt = 0;
+        int      type = 0;
+        int      start_off = 0;
+        int      stop_off = 0;
+        int      disk_layout[4];
+
+	if (!disk_layout_raw) {
+		gf_log (this->name, GF_LOG_CRITICAL,
+                        "error no layout on disk for merge");
+		return -1;
+	}
 
-	cnt  = ntoh32 (disk_layout[0]);
-	if (cnt != 1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"disk layout has invalid count %d", cnt);
+	GF_ASSERT (disk_layout_len == sizeof (disk_layout));
+
+        memcpy (disk_layout, disk_layout_raw, disk_layout_len);
+
+        cnt  = ntoh32 (disk_layout[0]);
+        if (cnt != 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "disk layout has invalid count %d", cnt);
+                return -1;
+        }
+
+        type = ntoh32 (disk_layout[1]);
+	switch (type) {
+        case DHT_HASH_TYPE_DM_USER:
+                gf_log (this->name, GF_LOG_DEBUG, "found user-set layout");
+                layout->type = type;
+                /* Fall through. */
+	case DHT_HASH_TYPE_DM:
+		break;
+        default:
+		gf_log (this->name, GF_LOG_CRITICAL,
+			"Catastrophic error layout with unknown type found %d",
+			disk_layout[1]);
 		return -1;
 	}
 
-	/* TODO: assert type is compatible */
-	type      = ntoh32 (disk_layout[1]);
-	start_off = ntoh32 (disk_layout[2]);
-	stop_off  = ntoh32 (disk_layout[3]);
+        start_off = ntoh32 (disk_layout[2]);
+        stop_off  = ntoh32 (disk_layout[3]);
 
-	layout->list[pos].start = start_off;
-	layout->list[pos].stop  = stop_off;
+        layout->list[pos].start = start_off;
+        layout->list[pos].stop  = stop_off;
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"merged to layout: %u - %u (type %d) from %s",
-		start_off, stop_off, type,
-		layout->list[pos].xlator->name);
+        gf_log (this->name, GF_LOG_TRACE,
+                "merged to layout: %u - %u (type %d) from %s",
+                start_off, stop_off, type,
+                layout->list[pos].xlator->name);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-		  int op_ret, int op_errno, dict_t *xattr)
+                  int op_ret, int op_errno, dict_t *xattr)
 {
-	int      i     = 0;
-	int      ret   = -1;
-	int      err   = -1;
-	int32_t *disk_layout = NULL;
-
-
-	if (op_ret != 0) {
-		err = op_errno;
-	}
-
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].xlator == NULL) {
-			layout->list[i].err    = err;
-			layout->list[i].xlator = subvol;
-			break;
-		}
-	}
-
-	if (op_ret != 0) {
-		ret = 0;
-		goto out;
-	}
-
-	if (xattr) {
-		/* during lookup and not mkdir */
-		ret = dict_get_ptr (xattr, "trusted.glusterfs.dht",
-				    VOID(&disk_layout));
-	}
-
-	if (ret != 0) {
-		layout->list[i].err = -1;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"missing disk layout on %s. err = %d",
-			subvol->name, err);
-		ret = 0;
-		goto out;
-	}
-
-	ret = dht_disk_layout_merge (this, layout, i, disk_layout);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"layout merge from subvolume %s failed",
-			subvol->name);
-		goto out;
-	}
-	layout->list[i].err = 0;
+        int         i     = 0;
+        int         ret   = -1;
+        int         err   = -1;
+        void       *disk_layout_raw = NULL;
+        int         disk_layout_len = 0;
+        dht_conf_t *conf = this->private;
+
+        if (op_ret != 0) {
+                err = op_errno;
+        }
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].xlator == NULL) {
+                        layout->list[i].err    = err;
+                        layout->list[i].xlator = subvol;
+                        break;
+                }
+        }
+
+        if (op_ret != 0) {
+                ret = 0;
+                goto out;
+        }
+
+        if (xattr) {
+                /* during lookup and not mkdir */
+                ret = dict_get_ptr_and_len (xattr, conf->xattr_name,
+					    &disk_layout_raw, &disk_layout_len);
+        }
+
+        if (ret != 0) {
+                layout->list[i].err = 0;
+                gf_log (this->name, GF_LOG_TRACE,
+                        "missing disk layout on %s. err = %d",
+                        subvol->name, err);
+                ret = 0;
+                goto out;
+        }
+
+        ret = dht_disk_layout_merge (this, layout, i, disk_layout_raw,
+				     disk_layout_len);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "layout merge from subvolume %s failed",
+                        subvol->name);
+                goto out;
+        }
+        layout->list[i].err = 0;
 
 out:
-	return ret;
+        return ret;
 }
 
 
 void
 dht_layout_entry_swap (dht_layout_t *layout, int i, int j)
 {
-	uint32_t  start_swap = 0;
-	uint32_t  stop_swap = 0;
-	xlator_t *xlator_swap = 0;
-	int       err_swap = 0;
-
-
-	start_swap  = layout->list[i].start;
-	stop_swap   = layout->list[i].stop;
-	xlator_swap = layout->list[i].xlator;
-	err_swap    = layout->list[i].err;
-
-	layout->list[i].start  = layout->list[j].start;
-	layout->list[i].stop   = layout->list[j].stop;
-	layout->list[i].xlator = layout->list[j].xlator;
-	layout->list[i].err    = layout->list[j].err;
-
-	layout->list[j].start  = start_swap;
-	layout->list[j].stop   = stop_swap;
-	layout->list[j].xlator = xlator_swap;
-	layout->list[j].err    = err_swap;
+        uint32_t  start_swap = 0;
+        uint32_t  stop_swap = 0;
+        xlator_t *xlator_swap = 0;
+        int       err_swap = 0;
+
+        start_swap  = layout->list[i].start;
+        stop_swap   = layout->list[i].stop;
+        xlator_swap = layout->list[i].xlator;
+        err_swap    = layout->list[i].err;
+
+        layout->list[i].start  = layout->list[j].start;
+        layout->list[i].stop   = layout->list[j].stop;
+        layout->list[i].xlator = layout->list[j].xlator;
+        layout->list[i].err    = layout->list[j].err;
+
+        layout->list[j].start  = start_swap;
+        layout->list[j].stop   = stop_swap;
+        layout->list[j].xlator = xlator_swap;
+        layout->list[j].err    = err_swap;
+}
+
+void
+dht_layout_range_swap (dht_layout_t *layout, int i, int j)
+{
+        uint32_t  start_swap = 0;
+        uint32_t  stop_swap = 0;
+
+        start_swap  = layout->list[i].start;
+        stop_swap   = layout->list[i].stop;
+
+        layout->list[i].start  = layout->list[j].start;
+        layout->list[i].stop   = layout->list[j].stop;
+
+        layout->list[j].start  = start_swap;
+        layout->list[j].stop   = stop_swap;
 }
 
 int64_t
 dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
 {
-	return (strcmp (layout->list[i].xlator->name, 
-			layout->list[j].xlator->name));
+        return (strcmp (layout->list[i].xlator->name,
+                        layout->list[j].xlator->name));
+}
+
+
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator)
+{
+        int i = 0;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (!strcmp (layout->list[i].xlator->name, xlator->name))
+                        return _gf_true;
+        }
+        return _gf_false;
 }
 
 int64_t
 dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
 {
-	int64_t diff = 0;
-
-	if (layout->list[i].err || layout->list[j].err)
-		diff = layout->list[i].err - layout->list[j].err;
-	else
-		diff = (int64_t) layout->list[i].start
-			- (int64_t) layout->list[j].start;
+        int64_t diff = 0;
+
+        /* swap zero'ed out layouts to front, if needed */
+        if (!layout->list[j].start && !layout->list[j].stop) {
+                diff = (int64_t) layout->list[i].stop
+                       - (int64_t) layout->list[j].stop;
+                       goto out;
+        }
+        if (layout->list[i].err || layout->list[j].err)
+                diff = layout->list[i].err - layout->list[j].err;
+        else
+                diff = (int64_t) layout->list[i].start
+                        - (int64_t) layout->list[j].start;
 
-	return diff;
+out:
+        return diff;
 }
 
 
 int
 dht_layout_sort (dht_layout_t *layout)
 {
-	int       i = 0;
-	int       j = 0;
-	int64_t   ret = 0;
-
-	/* TODO: O(n^2) -- bad bad */
-
-	for (i = 0; i < layout->cnt - 1; i++) {
-		for (j = i + 1; j < layout->cnt; j++) {
-			ret = dht_layout_entry_cmp (layout, i, j);
-			if (ret > 0)
-				dht_layout_entry_swap (layout, i, j);
-		}
-	}
+        int       i = 0;
+        int       j = 0;
+        int64_t   ret = 0;
+
+        /* TODO: O(n^2) -- bad bad */
 
-	return 0;
+        for (i = 0; i < layout->cnt - 1; i++) {
+                for (j = i + 1; j < layout->cnt; j++) {
+                        ret = dht_layout_entry_cmp (layout, i, j);
+                        if (ret > 0)
+                                dht_layout_entry_swap (layout, i, j);
+                }
+        }
+
+        return 0;
 }
 
 int
 dht_layout_sort_volname (dht_layout_t *layout)
 {
-	int       i = 0;
-	int       j = 0;
-	int64_t   ret = 0;
-
-	/* TODO: O(n^2) -- bad bad */
-
-	for (i = 0; i < layout->cnt - 1; i++) {
-		for (j = i + 1; j < layout->cnt; j++) {
-			ret = dht_layout_entry_cmp_volname (layout, i, j);
-			if (ret > 0)
-				dht_layout_entry_swap (layout, i, j);
-		}
-	}
+        int       i = 0;
+        int       j = 0;
+        int64_t   ret = 0;
+
+        /* TODO: O(n^2) -- bad bad */
 
-	return 0;
+        for (i = 0; i < layout->cnt - 1; i++) {
+                for (j = i + 1; j < layout->cnt; j++) {
+                        ret = dht_layout_entry_cmp_volname (layout, i, j);
+                        if (ret > 0)
+                                dht_layout_entry_swap (layout, i, j);
+                }
+        }
+
+        return 0;
 }
 
 
 int
 dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
-		      uint32_t *holes_p, uint32_t *overlaps_p,
-		      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p)
+                      uint32_t *holes_p, uint32_t *overlaps_p,
+                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                      uint32_t *no_space_p)
 {
-	dht_conf_t *conf = NULL;
-	uint32_t    holes    = 0;
-	uint32_t    overlaps = 0;
-	uint32_t    missing  = 0;
-	uint32_t    down     = 0;
-	uint32_t    misc     = 0;
-	uint32_t    hole_cnt = 0;
-	uint32_t    overlap_cnt = 0;
-	int         i = 0;
-	int         ret = 0;
-	uint32_t    prev_stop = 0;
-	uint32_t    last_stop = 0;
-	char        is_virgin = 1;
-
-
-	conf = this->private;
-
-	/* TODO: explain WTF is happening */
-
-	last_stop = layout->list[0].start - 1;
-	prev_stop = last_stop;
-
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err) {
-			switch (layout->list[i].err) {
-			case -1:
-			case ENOENT:
-				missing++;
-				break;
-			case ENOTCONN:
-				down++;
-				break;
-			default:
-				misc++;
-			}
-			continue;
-		}
-
-		is_virgin = 0;
-
-		if ((prev_stop + 1) < layout->list[i].start) {
-			hole_cnt++;
-			holes += (layout->list[i].start - (prev_stop + 1));
-		}
-
-		if ((prev_stop + 1) > layout->list[i].start) {
-			overlap_cnt++;
-			overlaps += ((prev_stop + 1) - layout->list[i].start);
-		}
-		prev_stop = layout->list[i].stop;
-	}
-
-	if ((last_stop - prev_stop) || is_virgin)
-	    hole_cnt++;
-	holes += (last_stop - prev_stop);
-
-	if (holes_p)
-		*holes_p = hole_cnt;
-
-	if (overlaps_p)
-		*overlaps_p = overlap_cnt;
-
-	if (missing_p)
-		*missing_p = missing;
-
-	if (down_p)
-		*down_p = down;
-
-	if (misc_p)
-		*misc_p = misc;
-
-	return ret;
+        uint32_t    overlaps = 0;
+        uint32_t    missing  = 0;
+        uint32_t    down     = 0;
+        uint32_t    misc     = 0;
+        uint32_t    hole_cnt = 0;
+        uint32_t    overlap_cnt = 0;
+        int         i = 0;
+        int         ret = 0;
+        uint32_t    prev_stop = 0;
+        uint32_t    last_stop = 0;
+        char        is_virgin = 1;
+        uint32_t    no_space  = 0;
+
+        /* TODO: explain what is happening */
+
+        last_stop = layout->list[0].start - 1;
+        prev_stop = last_stop;
+
+        for (i = 0; i < layout->cnt; i++) {
+                switch (layout->list[i].err) {
+                case -1:
+                case ENOENT:
+                        missing++;
+                        continue;
+                case ENOTCONN:
+                        down++;
+                        continue;
+                case ENOSPC:
+                        no_space++;
+                        continue;
+                case 0:
+                        /* if err == 0 and start == stop, then it is a non misc++;
+                         * participating subvolume(spread-cnt). Then, do not
+                         * check for anomalies. If start != stop, then treat it
+                         * as misc err */
+                        if (layout->list[i].start == layout->list[i].stop) {
+                                continue;
+                        }
+                        break;
+                default:
+                        misc++;
+                        continue;
+                 }
+
+                is_virgin = 0;
+
+                if ((prev_stop + 1) < layout->list[i].start) {
+                        hole_cnt++;
+                }
+
+                if ((prev_stop + 1) > layout->list[i].start) {
+                        overlap_cnt++;
+                        overlaps += ((prev_stop + 1) - layout->list[i].start);
+                }
+                prev_stop = layout->list[i].stop;
+        }
+
+        if ((last_stop - prev_stop) || is_virgin)
+                hole_cnt++;
+
+        if (holes_p)
+                *holes_p = hole_cnt;
+
+        if (overlaps_p)
+                *overlaps_p = overlap_cnt;
+
+        if (missing_p)
+                *missing_p = missing;
+
+        if (down_p)
+                *down_p = down;
+
+        if (misc_p)
+                *misc_p = misc;
+
+        if (no_space_p)
+                *no_space_p = no_space;
+
+        return ret;
 }
 
 
 int
 dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
 {
-	int          ret   = 0;
-	int          i = 0;
-	uint32_t     holes = 0;
-	uint32_t     overlaps = 0;
-	uint32_t     missing = 0;
-	uint32_t     down = 0;
-	uint32_t     misc = 0;
-
-
-	ret = dht_layout_sort (layout);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"sort failed?! how the ....");
-		goto out;
-	}
+        int          ret   = 0;
+        int          i = 0;
+        uint32_t     holes = 0;
+        uint32_t     overlaps = 0;
+        uint32_t     missing = 0;
+        uint32_t     down = 0;
+        uint32_t     misc = 0;
+
+        ret = dht_layout_sort (layout);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "sort failed?! how the ....");
+                goto out;
+        }
+
+        ret = dht_layout_anomalies (this, loc, layout,
+                                    &holes, &overlaps,
+                                    &missing, &down, &misc, NULL);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "error while finding anomalies in %s -- not good news",
+                        loc->path);
+                goto out;
+        }
+
+        if (holes || overlaps) {
+                if (missing == layout->cnt) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "directory %s looked up first time",
+                                loc->path);
+                } else {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "found anomalies in %s. holes=%d overlaps=%d",
+                                loc->path, holes, overlaps);
+                }
+                ret = -1;
+        }
+
+        for (i = 0; i < layout->cnt; i++) {
+                /* TODO During DHT selfheal rewrite (almost) find a better place
+                 * to detect this - probably in dht_layout_anomalies()
+                 */
+                if (layout->list[i].err > 0) {
+                        gf_log_callingfn (this->name, GF_LOG_DEBUG,
+                                          "path=%s err=%s on subvol=%s",
+                                          loc->path,
+                                          strerror (layout->list[i].err),
+                                          (layout->list[i].xlator ?
+                                           layout->list[i].xlator->name
+                                           : "<>"));
+                        if ((layout->list[i].err == ENOENT) && (ret >= 0)) {
+                                ret++;
+                        }
+                }
+        }
 
-	ret = dht_layout_anomalies (this, loc, layout,
-				    &holes, &overlaps,
-				    &missing, &down, &misc);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"error while finding anomalies in %s -- not good news",
-			loc->path);
-		goto out;
-	}
-
-	if (holes || overlaps) {
-		if (missing == layout->cnt) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"directory %s looked up first time",
-				loc->path);
-		} else {
-			gf_log (this->name, GF_LOG_ERROR,
-				"found anomalies in %s. holes=%d overlaps=%d",
-				loc->path, holes, overlaps);
-		}
-		ret = 1;
-	}
-
-	for (i = 0; i < layout->cnt; i++) {
-	/* TODO During DHT selfheal rewrite (almost) find a better place to 
-	 * detect this - probably in dht_layout_anomalies() 
-	 */
-		if (layout->list[i].err == ENOENT) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"path=%s ENOENT - directory entry"
-				" should be created in selfheal", loc->path);
-			ret = 1;
-			break;
-		}
-	}
 
 out:
-	return ret;
+        return ret;
 }
 
+int
+dht_dir_has_layout (dict_t *xattr, char *name)
+{
+
+        void     *disk_layout_raw = NULL;
+
+        return dict_get_ptr (xattr, name, &disk_layout_raw);
+}
 
 int
 dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-			 loc_t *loc, dict_t *xattr)
+                         loc_t *loc, dict_t *xattr)
 {
-	int       idx = 0;
-	int       pos = -1;
-	int       ret = -1;
-	int32_t  *disk_layout = NULL;
-	int32_t   count = -1;
-	uint32_t  start_off = -1;
-	uint32_t  stop_off = -1;
-
-
-	for (idx = 0; idx < layout->cnt; idx++) {
-		if (layout->list[idx].xlator == subvol) {
-			pos = idx;
-			break;
-		}
-	}
-	
-	if (pos == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"%s - no layout info for subvolume %s",
-			loc->path, subvol->name);
-		ret = 1;
-		goto out;
-	}
-	
-	if (xattr == NULL) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s - xattr dictionary is NULL",
-			loc->path);
-		ret = -1;
-		goto out;
-	}
+        int         idx = 0;
+        int         pos = -1;
+        int         ret = 0;
+        int         err = 0;
+        int         dict_ret = 0;
+        int32_t     disk_layout[4];
+        void       *disk_layout_raw = NULL;
+        int32_t     count = -1;
+        uint32_t    start_off = -1;
+        uint32_t    stop_off = -1;
+        dht_conf_t *conf = this->private;
+
+
+        for (idx = 0; idx < layout->cnt; idx++) {
+                if (layout->list[idx].xlator == subvol) {
+                        pos = idx;
+                        break;
+                }
+        }
+
+        if (pos == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s - no layout info for subvolume %s",
+                        loc->path, subvol->name);
+                ret = 1;
+                goto out;
+        }
+
+        err = layout->list[pos].err;
+
+        if (!xattr) {
+                if (err == 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%s - xattr dictionary is NULL",
+                                loc->path);
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        dict_ret = dict_get_ptr (xattr, conf->xattr_name,
+                                 &disk_layout_raw);
+
+        if (dict_ret < 0) {
+                if (err == 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%s - disk layout missing", loc->path);
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        memcpy (disk_layout, disk_layout_raw, sizeof (disk_layout));
+
+        count  = ntoh32 (disk_layout[0]);
+        if (count != 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s - disk layout has invalid count %d",
+                        loc->path, count);
+                ret = -1;
+                goto out;
+        }
+
+        start_off = ntoh32 (disk_layout[2]);
+        stop_off  = ntoh32 (disk_layout[3]);
+
+        if ((layout->list[pos].start != start_off)
+            || (layout->list[pos].stop != stop_off)) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "subvol: %s; inode layout - %"PRIu32" - %"PRIu32"; "
+                        "disk layout - %"PRIu32" - %"PRIu32,
+                        layout->list[pos].xlator->name,
+                        layout->list[pos].start, layout->list[pos].stop,
+                        start_off, stop_off);
+                ret = 1;
+        } else {
+                ret = 0;
+        }
+out:
+        return ret;
+}
 
-	ret = dict_get_ptr (xattr, "trusted.glusterfs.dht",
-			    VOID(&disk_layout));
-	
-	if (ret < 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s - disk layout missing", loc->path);
-		ret = -1;
-		goto out;
-	} 
-
-	count  = ntoh32 (disk_layout[0]);
-	if (count != 1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s - disk layout has invalid count %d",
-			loc->path, count);
-		ret = -1;
-		goto out;
-	}
 
-	start_off = ntoh32 (disk_layout[2]);
-	stop_off  = ntoh32 (disk_layout[3]);
-	
-	if ((layout->list[pos].start != start_off)
-	    || (layout->list[pos].stop != stop_off)) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"subvol: %s; inode layout - %"PRId32" - %"PRId32"; "
-			"disk layout - %"PRId32" - %"PRId32,
-			layout->list[pos].xlator->name,
-			layout->list[pos].start, layout->list[pos].stop,
-			start_off, stop_off);
-		ret = 1;
-	} else {
-		ret = 0;
-	}
+int
+dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode)
+{
+        dht_layout_t *layout = NULL;
+        int           ret = -1;
+        dht_conf_t   *conf = NULL;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        layout = dht_layout_for_subvol (this, subvol);
+        if (!layout) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no pre-set layout for subvolume %s",
+                        subvol ? subvol->name : "<nil>");
+                ret = -1;
+                goto out;
+        }
+
+        LOCK (&conf->layout_lock);
+        {
+                dht_inode_ctx_layout_set (inode, this, layout);
+        }
+        UNLOCK (&conf->layout_lock);
+
+        ret = 0;
 out:
-	return ret;
+        return ret;
 }
-
diff --git a/xlators/cluster/dht/src/dht-linkfile.c b/xlators/cluster/dht/src/dht-linkfile.c
index 9cc24ccf6..dbc9d0b3c 100644
--- a/xlators/cluster/dht/src/dht-linkfile.c
+++ b/xlators/cluster/dht/src/dht-linkfile.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -28,197 +19,310 @@
 #include "compat.h"
 #include "dht-common.h"
 
-
-
 int
-dht_linkfile_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int op_ret, int op_errno)
+dht_linkfile_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno,
+                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                         struct iatt *postparent)
 {
-	dht_local_t *local = NULL;
-
-
-	local = frame->local;
-	local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-				      local->linkfile.inode,
-				      &local->linkfile.stbuf);
-
-	return 0;
+        char          is_linkfile   = 0;
+        dht_conf_t   *conf          = NULL;
+        dht_local_t  *local         = NULL;
+        call_frame_t *prev          = NULL;
+
+        local = frame->local;
+        prev = cookie;
+        conf = this->private;
+
+        if (op_ret)
+                goto out;
+
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
+        if (!is_linkfile)
+                gf_log (this->name, GF_LOG_WARNING, "got non-linkfile %s:%s",
+                        prev->this->name, local->loc.path);
+out:
+        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
+                                      inode, stbuf, postparent, postparent,
+                                      xattr);
+        return 0;
 }
 
-
+#define is_equal(a, b) (a == b)
 int
 dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int op_ret, int op_errno,
-			 inode_t *inode, struct stat *stbuf)
+                         int op_ret, int op_errno, inode_t *inode,
+                         struct iatt *stbuf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
-	dict_t       *xattr = NULL;
-	data_t       *str_data = NULL;
-	int           ret = -1;
-
-	local = frame->local;
-	prev  = cookie;
-
-	if (op_ret == -1)
-		goto err;
-
-	xattr = get_new_dict ();
-	if (!xattr) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	local->linkfile.xattr = dict_ref (xattr);
-	local->linkfile.inode = inode_ref (inode);
-
-	str_data = str_to_data (local->linkfile.srcvol->name);
-	if (!str_data) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	ret = dict_set (xattr, "trusted.glusterfs.dht.linkto", str_data);
-	if (ret < 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"failed to initialize linkfile data");
-		op_errno = EINVAL;
-	}
-	str_data = NULL;
-
-	local->linkfile.stbuf = *stbuf;
-
-	STACK_WIND (frame, dht_linkfile_xattr_cbk,
-		    prev->this, prev->this->fops->setxattr,
-		    &local->linkfile.loc, local->linkfile.xattr, 0);
-
-	return 0;
-
-err:
-	if (str_data) {
-		data_destroy (str_data);
-		str_data = NULL;
-	}
-
-	local->linkfile.linkfile_cbk (frame, cookie, this,
-				      op_ret, op_errno, inode, stbuf);
-	return 0;
+        dht_local_t  *local = NULL;
+        xlator_t     *subvol = NULL;
+        call_frame_t *prev = NULL;
+        dict_t       *xattrs = NULL;
+        dht_conf_t   *conf = NULL;
+        int           ret = -1;
+
+        local = frame->local;
+
+        if (!op_ret)
+                local->linked = _gf_true;
+
+        FRAME_SU_UNDO (frame, dht_local_t);
+
+        if (op_ret && (op_errno == EEXIST)) {
+                conf = this->private;
+                prev = cookie;
+                subvol = prev->this;
+                if (!subvol)
+                        goto out;
+                xattrs = dict_new ();
+                if (!xattrs)
+                        goto out;
+                ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set linkto key");
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_linkfile_lookup_cbk, subvol,
+                            subvol->fops->lookup, &local->loc, xattrs);
+                if (xattrs)
+                        dict_unref (xattrs);
+                return 0;
+        }
+out:
+        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
+                                      inode, stbuf, preparent, postparent,
+                                      xdata);
+        if (xattrs)
+                dict_unref (xattrs);
+        return 0;
 }
 
 
 int
 dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
-		     xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
+                     xlator_t *this,
+                     xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
 {
-	dht_local_t *local = NULL;
-
-
-	local = frame->local;
-	local->linkfile.linkfile_cbk = linkfile_cbk;
-	local->linkfile.srcvol = tovol;
-	loc_copy (&local->linkfile.loc, loc);
+        dht_local_t *local = NULL;
+        dict_t      *dict = NULL;
+        int          need_unref = 0;
+        int          ret = 0;
+        dht_conf_t  *conf = this->private;
+
+        local = frame->local;
+        local->linkfile.linkfile_cbk = linkfile_cbk;
+        local->linkfile.srcvol = tovol;
+
+        local->linked = _gf_false;
+
+        dict = local->params;
+        if (!dict) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+                need_unref = 1;
+        }
+
+        if (!uuid_is_null (local->gfid)) {
+                ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
+                if (ret)
+                        gf_log ("dht-linkfile", GF_LOG_INFO,
+                                "%s: gfid set failed", loc->path);
+        }
+
+        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+        if (ret)
+                gf_log ("dht-linkfile", GF_LOG_INFO,
+                        "%s: internal-fop set failed", loc->path);
+
+        ret = dict_set_str (dict, conf->link_xattr_name, tovol->name);
+
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_INFO,
+                        "%s: failed to initialize linkfile data",
+                        loc->path);
+                goto out;
+        }
+
+        local->link_subvol = fromvol;
+        /* Always create as root:root. dht_linkfile_attr_heal fixes the
+         * ownsership */
+        FRAME_SU_DO (frame, dht_local_t);
+        STACK_WIND (frame, dht_linkfile_create_cbk,
+                    fromvol, fromvol->fops->mknod, loc,
+                    S_IFREG | DHT_LINKFILE_MODE, 0, 0, dict);
+
+        if (need_unref && dict)
+                dict_unref (dict);
+
+        return 0;
+out:
+        local->linkfile.linkfile_cbk (frame, NULL, frame->this, -1, ENOMEM,
+                                      loc->inode, NULL, NULL, NULL, NULL);
 
-	STACK_WIND (frame, dht_linkfile_create_cbk,
-		    fromvol, fromvol->fops->mknod, loc,
-		    S_IFREG | DHT_LINKFILE_MODE, 0);
+        if (need_unref && dict)
+                dict_unref (dict);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int32_t op_ret, int32_t op_errno)
+                         int32_t op_ret, int32_t op_errno,
+                         struct iatt *preparent, struct iatt *postparent,
+                         dict_t *xdata)
 {
-	dht_local_t   *local = NULL;
-	call_frame_t  *prev = NULL;
-	xlator_t      *subvol = NULL;
+        dht_local_t   *local = NULL;
+        call_frame_t  *prev = NULL;
+        xlator_t      *subvol = NULL;
 
-	local = frame->local;
-	prev = cookie;
-	subvol = prev->this;
+        local = frame->local;
+        prev = cookie;
+        subvol = prev->this;
 
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"unlinking linkfile %s on %s failed (%s)",
-			local->loc.path, subvol->name, strerror (op_errno));
-	}
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "unlinking linkfile %s on %s failed (%s)",
+                        local->loc.path, subvol->name, strerror (op_errno));
+        }
 
-	DHT_STACK_DESTROY (frame);
+        DHT_STACK_DESTROY (frame);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
-		     xlator_t *subvol, loc_t *loc)
+                     xlator_t *subvol, loc_t *loc)
 {
-	call_frame_t *unlink_frame = NULL;
-	dht_local_t  *unlink_local = NULL;
-
-	unlink_frame = copy_frame (frame);
-	if (!unlink_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-	unlink_local = dht_local_init (unlink_frame);
-	if (!unlink_local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-	loc_copy (&unlink_local->loc, loc);
-
-	STACK_WIND (unlink_frame, dht_linkfile_unlink_cbk,
-		    subvol, subvol->fops->unlink,
-		    &unlink_local->loc);
-
-	return 0;
+        call_frame_t *unlink_frame = NULL;
+        dht_local_t  *unlink_local = NULL;
+
+        unlink_frame = copy_frame (frame);
+        if (!unlink_frame) {
+                goto err;
+        }
+
+        /* Using non-fop value here, as anyways, 'local->fop' is not used in
+           this particular case */
+        unlink_local = dht_local_init (unlink_frame, loc, NULL,
+                                       GF_FOP_MAXVALUE);
+        if (!unlink_local) {
+                goto err;
+        }
+
+        STACK_WIND (unlink_frame, dht_linkfile_unlink_cbk,
+                    subvol, subvol->fops->unlink,
+                    &unlink_local->loc, 0, NULL);
+
+        return 0;
 err:
-	if (unlink_frame)
-		DHT_STACK_DESTROY (unlink_frame);
+        if (unlink_frame)
+                DHT_STACK_DESTROY (unlink_frame);
 
-	return -1;
+        return -1;
 }
 
 
 xlator_t *
-dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct stat *stbuf,
-		     dict_t *xattr)
+dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
+                     dict_t *xattr)
 {
-	dht_conf_t *conf = NULL;
-	xlator_t   *subvol = NULL;
-	void       *volname = NULL;
-	int         i = 0, ret = 0;
-
+        dht_conf_t *conf = NULL;
+        xlator_t   *subvol = NULL;
+        void       *volname = NULL;
+        int         i = 0, ret = 0;
 
-	conf = this->private;
+        conf = this->private;
 
-	if (!xattr)
-		goto out;
+        if (!xattr)
+                goto out;
 
-	ret = dict_get_ptr (xattr, "trusted.glusterfs.dht.linkto", &volname);
+        ret = dict_get_ptr (xattr, conf->link_xattr_name, &volname);
 
-	if ((-1 == ret) || !volname)
-		goto out;
+        if ((-1 == ret) || !volname)
+                goto out;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (strcmp (conf->subvolumes[i]->name, (char *)volname) == 0) {
-			subvol = conf->subvolumes[i];
-			break;
-		}
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (strcmp (conf->subvolumes[i]->name, (char *)volname) == 0) {
+                        subvol = conf->subvolumes[i];
+                        break;
+                }
+        }
 
 out:
-	return subvol;
+        return subvol;
+}
+
+int
+dht_linkfile_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, struct iatt *statpre,
+                          struct iatt *statpost, dict_t *xdata)
+{
+        dht_local_t *local = NULL;
+        loc_t *loc = NULL;
+
+        local = frame->local;
+        loc = &local->loc;
+
+        if (op_ret)
+                gf_log (this->name, GF_LOG_ERROR, "setattr of uid/gid on %s"
+                        " :<gfid:%s> failed (%s)",
+                        (loc->path? loc->path: "NULL"),
+                        uuid_utoa(local->gfid), strerror(op_errno));
+
+        DHT_STACK_DESTROY (frame);
+
+       return 0;
 }
 
+int
+dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this)
+{
+        int     ret = -1;
+        call_frame_t *copy = NULL;
+        dht_local_t  *local = NULL;
+        dht_local_t  *copy_local = NULL;
+        xlator_t     *subvol = NULL;
+        struct iatt   stbuf = {0,};
+
+        local = frame->local;
+
+        GF_VALIDATE_OR_GOTO ("dht", local, out);
+        GF_VALIDATE_OR_GOTO ("dht", local->link_subvol, out);
+
+        if (local->stbuf.ia_type == IA_INVAL)
+                return 0;
+
+        uuid_copy (local->loc.gfid, local->stbuf.ia_gfid);
 
+        copy = copy_frame (frame);
+
+        if (!copy)
+                goto out;
+
+       copy_local = dht_local_init (copy, &local->loc, NULL, 0);
+
+       if (!copy_local)
+               goto out;
+
+        stbuf = local->stbuf;
+        subvol = local->link_subvol;
+
+        copy->local = copy_local;
+
+        FRAME_SU_DO (copy, dht_local_t);
+
+        STACK_WIND (copy, dht_linkfile_setattr_cbk, subvol,
+                    subvol->fops->setattr, &copy_local->loc,
+                    &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h
new file mode 100644
index 000000000..e893eb48f
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-mem-types.h
@@ -0,0 +1,35 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __DHT_MEM_TYPES_H__
+#define __DHT_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_dht_mem_types_ {
+        gf_dht_mt_dht_du_t = gf_common_mt_end + 1,
+        gf_dht_mt_dht_conf_t,
+        gf_dht_mt_char,
+        gf_dht_mt_int32_t,
+        gf_dht_mt_xlator_t,
+        gf_dht_mt_dht_layout_t,
+        gf_switch_mt_dht_conf_t,
+        gf_switch_mt_dht_du_t,
+        gf_switch_mt_switch_sched_array,
+        gf_switch_mt_switch_struct,
+        gf_dht_mt_subvol_time,
+        gf_dht_mt_loc_t,
+        gf_defrag_info_mt,
+        gf_dht_mt_inode_ctx_t,
+        gf_dht_mt_ctx_stat_time_t,
+        gf_dht_mt_end
+};
+#endif
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
new file mode 100644
index 000000000..bcb19f23e
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -0,0 +1,1815 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "dht-common.h"
+#include "xlator.h"
+#include <fnmatch.h>
+
+#define GF_DISK_SECTOR_SIZE             512
+#define DHT_REBALANCE_PID               4242 /* Change it if required */
+#define DHT_REBALANCE_BLKSIZE           (128 * 1024)
+
+static int
+dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
+                      int32_t size, off_t offset, struct iobref *iobref)
+{
+        int i            = 0;
+        int ret          = -1;
+        int start_idx    = 0;
+        int tmp_offset   = 0;
+        int write_needed = 0;
+        int buf_len      = 0;
+        int size_pending = 0;
+        char *buf        = NULL;
+
+        /* loop through each vector */
+        for (i = 0; i < count; i++) {
+                buf = vec[i].iov_base;
+                buf_len = vec[i].iov_len;
+
+                for (start_idx = 0; (start_idx + GF_DISK_SECTOR_SIZE) <= buf_len;
+                     start_idx += GF_DISK_SECTOR_SIZE) {
+
+                        if (mem_0filled (buf + start_idx, GF_DISK_SECTOR_SIZE) != 0) {
+                                write_needed = 1;
+                                continue;
+                        }
+
+                        if (write_needed) {
+                                ret = syncop_write (to, fd, (buf + tmp_offset),
+                                                    (start_idx - tmp_offset),
+                                                    (offset + tmp_offset),
+                                                    iobref, 0);
+                                /* 'path' will be logged in calling function */
+                                if (ret < 0) {
+                                        gf_log (THIS->name, GF_LOG_WARNING,
+                                                "failed to write (%s)",
+                                                strerror (errno));
+                                        goto out;
+                                }
+
+                                write_needed = 0;
+                        }
+                        tmp_offset = start_idx + GF_DISK_SECTOR_SIZE;
+                }
+
+                if ((start_idx < buf_len) || write_needed) {
+                        /* This means, last chunk is not yet written.. write it */
+                        ret = syncop_write (to, fd, (buf + tmp_offset),
+                                            (buf_len - tmp_offset),
+                                            (offset + tmp_offset), iobref, 0);
+                        if (ret < 0) {
+                                /* 'path' will be logged in calling function */
+                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        "failed to write (%s)",
+                                        strerror (errno));
+                                goto out;
+                        }
+                }
+
+                size_pending = (size - buf_len);
+                if (!size_pending)
+                        break;
+        }
+
+        ret = size;
+out:
+        return ret;
+
+}
+
+int32_t
+gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
+                           struct iatt *stbuf)
+{
+        int32_t                 ret             = -1;
+        xlator_t               *cached_subvol   = NULL;
+        xlator_t               *hashed_subvol   = NULL;
+        xlator_t               *linkto_subvol   = NULL;
+        data_t                 *data            = NULL;
+        struct iatt             iatt            = {0,};
+        int32_t                 op_errno        = 0;
+        dht_conf_t             *conf            = NULL;
+
+        GF_VALIDATE_OR_GOTO ("defrag", loc, out);
+        GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);
+        GF_VALIDATE_OR_GOTO ("defrag", stbuf, out);
+        GF_VALIDATE_OR_GOTO ("defrag", this, out);
+        GF_VALIDATE_OR_GOTO ("defrag", xattrs, out);
+        GF_VALIDATE_OR_GOTO ("defrag", this->private, out);
+
+        conf = this->private;
+
+        if (uuid_is_null (loc->pargfid)) {
+                gf_log ("", GF_LOG_ERROR, "loc->pargfid is NULL for "
+                        "%s", loc->path);
+                goto out;
+        }
+
+        if (uuid_is_null (loc->gfid)) {
+                gf_log ("", GF_LOG_ERROR, "loc->gfid is NULL for "
+                        "%s", loc->path);
+                goto out;
+        }
+
+        cached_subvol = dht_subvol_get_cached (this, loc->inode);
+        if (!cached_subvol) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get cached subvol"
+                        " for %s on %s", loc->name, this->name);
+                goto out;
+        }
+
+        hashed_subvol = dht_subvol_get_hashed (this, loc);
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get hashed subvol"
+                        " for %s on %s", loc->name, this->name);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s "
+                "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid),
+                cached_subvol->name, hashed_subvol->name);
+        data = dict_get (xattrs, conf->link_xattr_name);
+        /* set linkto on cached -> hashed if not present, else link it */
+        if (!data) {
+                ret = dict_set_str (xattrs, conf->link_xattr_name,
+                                    hashed_subvol->name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "linkto xattr in dict for %s", loc->name);
+                        goto out;
+                }
+
+                ret = syncop_setxattr (cached_subvol, loc, xattrs, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Linkto setxattr "
+                                "failed %s -> %s (%s)", cached_subvol->name,
+                                loc->name, strerror (errno));
+                        goto out;
+                }
+                goto out;
+        } else {
+                linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattrs);
+                if (!linkto_subvol) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "linkto subvol for %s", loc->name);
+                } else {
+                        hashed_subvol = linkto_subvol;
+                }
+
+                ret = syncop_link (hashed_subvol, loc, loc);
+                if  (ret) {
+                        op_errno = errno;
+                        gf_log (this->name, GF_LOG_ERROR, "link of %s -> %s"
+                                " failed on  subvol %s (%s)", loc->name,
+                                uuid_utoa(loc->gfid),
+                                hashed_subvol->name, strerror (op_errno));
+                        if (op_errno != EEXIST)
+                                goto out;
+                }
+        }
+        ret = syncop_lookup (hashed_subvol, loc, NULL, &iatt, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed lookup %s on %s (%s)"
+                        , loc->name, hashed_subvol->name, strerror (errno));
+                goto out;
+        }
+
+        if (iatt.ia_nlink == stbuf->ia_nlink) {
+                ret = dht_migrate_file (this, loc, cached_subvol, hashed_subvol,
+                                        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS);
+                if (ret)
+                        goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+
+static inline int
+__is_file_migratable (xlator_t *this, loc_t *loc,
+                      struct iatt *stbuf, dict_t *xattrs, int flags)
+{
+        int ret = -1;
+
+        if (IA_ISDIR (stbuf->ia_type)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: migrate-file called on directory", loc->path);
+                ret = -1;
+                goto out;
+        }
+
+        if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
+                ret = 0;
+                goto out;
+        }
+        if (stbuf->ia_nlink > 1) {
+                /* support for decomission */
+                if (flags == GF_DHT_MIGRATE_HARDLINK) {
+                        ret = gf_defrag_handle_hardlink (this, loc,
+                                                         xattrs, stbuf);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "%s: failed to migrate file with link",
+                                        loc->path);
+                        }
+                } else {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: file has hardlinks", loc->path);
+                }
+                ret = ENOTSUP;
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static inline int
+__dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,
+                                 dict_t *dict, fd_t **dst_fd, dict_t *xattr)
+{
+        xlator_t    *this = NULL;
+        int          ret  = -1;
+        fd_t        *fd   = NULL;
+        struct iatt  new_stbuf = {0,};
+        dht_conf_t  *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+
+        ret = dict_set_static_bin (dict, "gfid-req", stbuf->ia_gfid, 16);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set gfid in dict for create", loc->path);
+                goto out;
+        }
+
+        ret = dict_set_str (dict, conf->link_xattr_name, from->name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set gfid in dict for create", loc->path);
+                goto out;
+        }
+
+        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: fd create failed (destination) (%s)",
+                        loc->path, strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_lookup (to, loc, NULL, &new_stbuf, NULL, NULL);
+        if (!ret) {
+                /* File exits in the destination, check if gfid matches */
+                if (uuid_compare (stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "file %s exits in %s with different gfid",
+                                loc->path, to->name);
+                        fd_unref (fd);
+                        goto out;
+                }
+        }
+        if ((ret == -1) && (errno != ENOENT)) {
+                /* File exists in destination, but not accessible */
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "%s: failed to lookup file (%s)",
+                        loc->path, strerror (errno));
+                goto out;
+        }
+
+        /* Create the destination with LINKFILE mode, and linkto xattr,
+           if the linkfile already exists, it will just open the file */
+        ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd,
+                             dict, &new_stbuf);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create %s on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+                goto out;
+        }
+
+        ret = syncop_fsetxattr (to, fd, xattr, 0);
+        if (ret == -1)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set xattr on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+
+        ret = syncop_ftruncate (to, fd, stbuf->ia_size);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "ftruncate failed for %s on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+
+        ret = syncop_fsetattr (to, fd, stbuf,
+                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                                NULL, NULL);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "chown failed for %s on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+
+        if (dst_fd)
+                *dst_fd = fd;
+
+        /* success */
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static inline int
+__dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
+                        struct iatt *stbuf, int flag)
+{
+        struct statvfs  src_statfs = {0,};
+        struct statvfs  dst_statfs = {0,};
+        int             ret        = -1;
+        xlator_t       *this       = NULL;
+
+        uint64_t        src_statfs_blocks = 1;
+        uint64_t        dst_statfs_blocks = 1;
+
+        this = THIS;
+
+        ret = syncop_statfs (from, loc, &src_statfs);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get statfs of %s on %s (%s)",
+                        loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+        ret = syncop_statfs (to, loc, &dst_statfs);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get statfs of %s on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+                goto out;
+        }
+
+        /* if force option is given, do not check for space @ dst.
+         * Check only if space is avail for the file */
+        if (flag != GF_DHT_MIGRATE_DATA)
+                goto check_avail_space;
+
+        /* Check:
+           During rebalance `migrate-data` - Destination subvol experiences
+           a `reduction` in 'blocks' of free space, at the same time source
+           subvol gains certain 'blocks' of free space. A valid check is
+           necessary here to avoid errorneous move to destination where
+           the space could be scantily available.
+         */
+        if (stbuf) {
+                dst_statfs_blocks = ((dst_statfs.f_bavail *
+                                      dst_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                src_statfs_blocks = ((src_statfs.f_bavail *
+                                      src_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                if ((dst_statfs_blocks - stbuf->ia_blocks) <
+                    (src_statfs_blocks + stbuf->ia_blocks)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "data movement attempted from node (%s) with"
+                                " higher disk space to a node (%s) with "
+                                "lesser disk space (%s)", from->name,
+                                to->name, loc->path);
+
+                        /* this is not a 'failure', but we don't want to
+                           consider this as 'success' too :-/ */
+                        ret = 1;
+                        goto out;
+                }
+        }
+check_avail_space:
+        if (((dst_statfs.f_bavail * dst_statfs.f_bsize) /
+              GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "data movement attempted from node (%s) with "
+                        "to node (%s) which does not have required free space"
+                        " for %s", from->name, to->name, loc->path);
+                ret = 1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static inline int
+__dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
+                             uint64_t ia_size, int hole_exists)
+{
+        int            ret    = 0;
+        int            count  = 0;
+        off_t          offset = 0;
+        struct iovec  *vector = NULL;
+        struct iobref *iobref = NULL;
+        uint64_t       total  = 0;
+        size_t         read_size = 0;
+
+        /* if file size is '0', no need to enter this loop */
+        while (total < ia_size) {
+                read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) ?
+                             DHT_REBALANCE_BLKSIZE : (ia_size - total));
+                ret = syncop_readv (from, src, read_size,
+                                    offset, 0, &vector, &count, &iobref);
+                if (!ret || (ret < 0)) {
+                        break;
+                }
+
+                if (hole_exists)
+                        ret = dht_write_with_holes (to, dst, vector, count,
+                                                    ret, offset, iobref);
+                else
+                        ret = syncop_writev (to, dst, vector, count,
+                                             offset, iobref, 0);
+                if (ret < 0) {
+                        break;
+                }
+                offset += ret;
+                total += ret;
+
+                GF_FREE (vector);
+                if (iobref)
+                        iobref_unref (iobref);
+                iobref = NULL;
+                vector = NULL;
+        }
+        if (iobref)
+                iobref_unref (iobref);
+        GF_FREE (vector);
+
+        if (ret >= 0)
+                ret = 0;
+
+        return ret;
+}
+
+
+static inline int
+__dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
+                               struct iatt *stbuf, fd_t **src_fd)
+{
+        int          ret  = 0;
+        fd_t        *fd   = NULL;
+        dict_t      *dict = NULL;
+        xlator_t    *this = NULL;
+        struct iatt  iatt = {0,};
+        dht_conf_t  *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+
+        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: fd create failed (source)", loc->path);
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_open (from, loc, O_RDWR, fd);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to open file %s on %s (%s)",
+                        loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+        ret = -1;
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_str (dict, conf->link_xattr_name, to->name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set xattr in dict for %s (linkto:%s)",
+                        loc->path, to->name);
+                goto out;
+        }
+
+        /* Once the migration starts, the source should have 'linkto' key set
+           to show which is the target, so other clients can work around it */
+        ret = syncop_setxattr (from, loc, dict, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set xattr on %s in %s (%s)",
+                        loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+        /* mode should be (+S+T) to indicate migration is in progress */
+        iatt.ia_prot = stbuf->ia_prot;
+        iatt.ia_type = stbuf->ia_type;
+        iatt.ia_prot.sticky = 1;
+        iatt.ia_prot.sgid = 1;
+
+        ret = syncop_setattr (from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set mode on %s in %s (%s)",
+                        loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+        if (src_fd)
+                *src_fd = fd;
+
+        /* success */
+        ret = 0;
+out:
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+int
+migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
+                       struct iatt *buf)
+{
+        int          ret      = -1;
+        dict_t      *rsp_dict = NULL;
+        dict_t      *dict     = NULL;
+        char        *link     = NULL;
+        struct iatt  stbuf    = {0,};
+        dht_conf_t  *conf     = this->private;
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set 'linkto' key in dict", loc->path);
+                goto out;
+        }
+
+        /* check in the destination if the file is link file */
+        ret = syncop_lookup (to, loc, dict, &stbuf, &rsp_dict, NULL);
+        if ((ret == -1) && (errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s: lookup failed (%s)",
+                        loc->path, strerror (errno));
+                goto out;
+        }
+
+        /* we no more require this key */
+        dict_del (dict, conf->link_xattr_name);
+
+        /* file exists in target node, only if it is 'linkfile' its valid,
+           otherwise, error out */
+        if (!ret) {
+                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict,
+                                        conf->link_xattr_name)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: file exists in destination", loc->path);
+                        ret = -1;
+                        goto out;
+                }
+
+                /* as file is linkfile, delete it */
+                ret = syncop_unlink (to, loc);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to delete the linkfile (%s)",
+                                loc->path, strerror (errno));
+                        goto out;
+                }
+        }
+
+        /* Set the gfid of the source file in dict */
+        ret = dict_set_static_bin (dict, "gfid-req", buf->ia_gfid, 16);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set gfid in dict for create", loc->path);
+                goto out;
+        }
+
+        /* Create the file in target */
+        if (IA_ISLNK (buf->ia_type)) {
+                /* Handle symlinks separately */
+                ret = syncop_readlink (from, loc, &link, buf->ia_size);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: readlink on symlink failed (%s)",
+                                loc->path, strerror (errno));
+                        goto out;
+                }
+
+                ret = syncop_symlink (to, loc, link, dict, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: creating symlink failed (%s)",
+                                loc->path, strerror (errno));
+                        goto out;
+                }
+
+                goto done;
+        }
+
+        ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot,
+                                                      buf->ia_type),
+                            makedev (ia_major (buf->ia_rdev),
+                                     ia_minor (buf->ia_rdev)), dict, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "%s: mknod failed (%s)",
+                        loc->path, strerror (errno));
+                goto out;
+        }
+
+done:
+        ret = syncop_setattr (to, loc, buf,
+                              (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
+                               GF_SET_ATTR_MODE), NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+        }
+
+        ret = syncop_unlink (from, loc);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "%s: unlink failed (%s)",
+                        loc->path, strerror (errno));
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        return ret;
+}
+
+/*
+  return values:
+
+   -1 : failure
+    0 : successfully migrated data
+    1 : not a failure, but we can't migrate data as of now
+*/
+int
+dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                  int flag)
+{
+        int             ret            = -1;
+        struct iatt     new_stbuf      = {0,};
+        struct iatt     stbuf          = {0,};
+        struct iatt     empty_iatt     = {0,};
+        ia_prot_t       src_ia_prot    = {0,};
+        fd_t           *src_fd         = NULL;
+        fd_t           *dst_fd         = NULL;
+        dict_t         *dict           = NULL;
+        dict_t         *xattr          = NULL;
+        dict_t         *xattr_rsp      = NULL;
+        int             file_has_holes = 0;
+        dht_conf_t     *conf           = this->private;
+
+        gf_log (this->name, GF_LOG_INFO, "%s: attempting to move from %s to %s",
+                loc->path, from->name, to->name);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set 'linkto' key in dict", loc->path);
+                goto out;
+        }
+
+        /* Phase 1 - Data migration is in progress from now on */
+        ret = syncop_lookup (from, loc, dict, &stbuf, &xattr_rsp, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: lookup failed on %s (%s)",
+                        loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+        /* we no more require this key */
+        dict_del (dict, conf->link_xattr_name);
+
+        /* preserve source mode, so set the same to the destination */
+        src_ia_prot = stbuf.ia_prot;
+
+        /* Check if file can be migrated */
+        ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag);
+        if (ret)
+                goto out;
+
+        /* Take care of the special files */
+        if (!IA_ISREG (stbuf.ia_type)) {
+                /* Special files */
+                ret = migrate_special_files (this, from, to, loc, &stbuf);
+                goto out;
+        }
+
+        /* TODO: move all xattr related operations to fd based operations */
+        ret = syncop_listxattr (from, loc, &xattr);
+        if (ret == -1)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get xattr from %s (%s)",
+                        loc->path, from->name, strerror (errno));
+
+        /* create the destination, with required modes/xattr */
+        ret = __dht_rebalance_create_dst_file (to, from, loc, &stbuf,
+                                               dict, &dst_fd, xattr);
+        if (ret)
+                goto out;
+
+        ret = __dht_check_free_space (to, from, loc, &stbuf, flag);
+        if (ret) {
+                goto out;
+        }
+
+        /* Open the source, and also update mode/xattr */
+        ret = __dht_rebalance_open_src_file (from, to, loc, &stbuf, &src_fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to open %s on %s",
+                        loc->path, from->name);
+                goto out;
+        }
+
+
+        ret = syncop_fstat (from, src_fd, &stbuf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to lookup %s on %s (%s)",
+                        loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+        /* Try to preserve 'holes' while migrating data */
+        if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
+                file_has_holes = 1;
+
+        /* All I/O happens in this function */
+        ret = __dht_rebalance_migrate_data (from, to, src_fd, dst_fd,
+					    stbuf.ia_size, file_has_holes);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: failed to migrate data",
+                        loc->path);
+                /* reset the destination back to 0 */
+                ret = syncop_ftruncate (to, dst_fd, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to reset target size back to 0 (%s)",
+                                loc->path, strerror (errno));
+                }
+
+                ret = -1;
+                goto out;
+        }
+
+        /* TODO: Sync the locks */
+
+        ret = syncop_fsync (to, dst_fd, 0);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to fsync on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+
+
+        /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
+
+        ret = syncop_fstat (from, src_fd, &new_stbuf);
+        if (ret < 0) {
+                /* Failed to get the stat info */
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to fstat file %s on %s (%s)",
+                        loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+        /* source would have both sticky bit and sgid bit set, reset it to 0,
+           and set the source permission on destination, if it was not set
+           prior to setting rebalance-modes in source  */
+        if (!src_ia_prot.sticky)
+                new_stbuf.ia_prot.sticky = 0;
+
+        if (!src_ia_prot.sgid)
+                new_stbuf.ia_prot.sgid = 0;
+
+        /* TODO: if the source actually had sticky bit, or sgid bit set,
+           we are not handling it */
+
+        ret = syncop_fsetattr (to, dst_fd, &new_stbuf,
+                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
+                                GF_SET_ATTR_MODE), NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+                goto out;
+        }
+
+        /* Because 'futimes' is not portable */
+        ret = syncop_setattr (to, loc, &new_stbuf,
+                              (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME),
+                              NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+        }
+
+        /* Make the source as a linkfile first before deleting it */
+        empty_iatt.ia_prot.sticky = 1;
+        ret = syncop_fsetattr (from, src_fd, &empty_iatt,
+                               GF_SET_ATTR_MODE, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,             \
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+       /* Free up the data blocks on the source node, as the whole
+           file is migrated */
+        ret = syncop_ftruncate (from, src_fd, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform truncate on %s (%s)",
+                        loc->path, from->name, strerror (errno));
+        }
+
+        /* remove the 'linkto' xattr from the destination */
+        ret = syncop_fremovexattr (to, dst_fd, conf->link_xattr_name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform removexattr on %s (%s)",
+                        loc->path, to->name, strerror (errno));
+        }
+
+        /* Do a stat and check the gfid before unlink */
+        ret = syncop_stat (from, loc, &empty_iatt);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to do a stat on %s (%s)",
+                        loc->path, from->name, strerror (errno));
+                goto out;
+        }
+
+        if (uuid_compare (empty_iatt.ia_gfid, loc->gfid) == 0) {
+                /* take out the source from namespace */
+                ret = syncop_unlink (from, loc);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to perform unlink on %s (%s)",
+                                loc->path, from->name, strerror (errno));
+                        goto out;
+                }
+        }
+
+        ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: failed to lookup the file on subvolumes (%s)",
+                        loc->path, strerror (errno));
+        }
+
+        gf_log (this->name, GF_LOG_INFO,
+                "completed migration of %s from subvolume %s to %s",
+                loc->path, from->name, to->name);
+
+        ret = 0;
+out:
+        if (dict)
+                dict_unref (dict);
+
+        if (xattr)
+                dict_unref (xattr);
+        if (xattr_rsp)
+                dict_unref (xattr_rsp);
+
+        if (dst_fd)
+                syncop_close (dst_fd);
+        if (src_fd)
+                syncop_close (src_fd);
+
+        return ret;
+}
+
+static int
+rebalance_task (void *data)
+{
+        int           ret   = -1;
+        dht_local_t  *local = NULL;
+        call_frame_t *frame = NULL;
+
+        frame = data;
+
+        local = frame->local;
+
+        /* This function is 'synchrounous', hence if it returns,
+           we are done with the task */
+        ret = dht_migrate_file (THIS, &local->loc, local->rebalance.from_subvol,
+                                local->rebalance.target_node, local->flags);
+
+        return ret;
+}
+
+static int
+rebalance_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
+{
+        int           ret        = -1;
+        uint64_t      layout_int = 0;
+        dht_layout_t *layout     = 0;
+        xlator_t     *this       = NULL;
+        dht_local_t  *local      = NULL;
+        int32_t       op_errno   = EINVAL;
+
+        this = THIS;
+        local = sync_frame->local;
+
+        if (!op_ret) {
+                /* Make sure we have valid 'layout' in inode ctx
+                   after the operation */
+                ret = inode_ctx_del (local->loc.inode, this, &layout_int);
+                if (!ret && layout_int) {
+                        layout = (dht_layout_t *)(long)layout_int;
+                        dht_layout_unref (this, layout);
+                }
+
+                ret = dht_layout_preset (this, local->rebalance.target_node,
+                                         local->loc.inode);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set inode ctx", local->loc.path);
+        }
+
+        if (op_ret == -1) {
+                /* Failure of migration process, mostly due to write process.
+                   as we can't preserve the exact errno, lets say there was
+                   no space to migrate-data
+                */
+                op_errno = ENOSPC;
+        }
+
+        if (op_ret == 1) {
+                /* migration didn't happen, but is not a failure, let the user
+                   understand that he doesn't have permission to migrate the
+                   file.
+                */
+                op_ret = -1;
+                op_errno = EPERM;
+        }
+
+        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno, NULL);
+        return 0;
+}
+
+int
+dht_start_rebalance_task (xlator_t *this, call_frame_t *frame)
+{
+        int         ret     = -1;
+
+        ret = synctask_new (this->ctx->env, rebalance_task,
+                            rebalance_task_completion,
+                            frame, frame);
+        return ret;
+}
+
+int
+gf_listener_stop (xlator_t *this)
+{
+        glusterfs_ctx_t  *ctx = NULL;
+        cmd_args_t       *cmd_args = NULL;
+        int              ret = 0;
+
+        ctx = this->ctx;
+        GF_ASSERT (ctx);
+        cmd_args = &ctx->cmd_args;
+        if (cmd_args->sock_file) {
+                ret = unlink (cmd_args->sock_file);
+                if (ret && (ENOENT == errno)) {
+                        ret = 0;
+                }
+        }
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink listener "
+                        "socket %s, error: %s", cmd_args->sock_file,
+                        strerror (errno));
+        }
+        return ret;
+}
+
+void
+dht_build_root_inode (xlator_t *this, inode_t **inode)
+{
+        inode_table_t    *itable        = NULL;
+        uuid_t            root_gfid     = {0, };
+
+        itable = inode_table_new (0, this);
+        if (!itable)
+                return;
+
+        root_gfid[15] = 1;
+        *inode = inode_find (itable, root_gfid);
+}
+
+void
+dht_build_root_loc (inode_t *inode, loc_t *loc)
+{
+        loc->path = "/";
+        loc->inode = inode;
+        loc->inode->ia_type = IA_IFDIR;
+        memset (loc->gfid, 0, 16);
+        loc->gfid[15] = 1;
+}
+
+
+/* return values: 1 -> error, bug ignore and continue
+                  0 -> proceed
+                 -1 -> error, handle it */
+int32_t
+gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
+{
+        /* if errno is not ENOSPC or ENOTCONN, we can still continue
+           with rebalance process */
+        if ((errno != ENOSPC) || (errno != ENOTCONN))
+                return 1;
+
+        if (errno == ENOTCONN) {
+                /* Most probably mount point went missing (mostly due
+                   to a brick down), say rebalance failure to user,
+                   let him restart it if everything is fine */
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+                return -1;
+        }
+
+        if (errno == ENOSPC) {
+                /* rebalance process itself failed, may be
+                   remote brick went down, or write failed due to
+                   disk full etc etc.. */
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+                return -1;
+        }
+
+        return 0;
+}
+
+static gf_boolean_t
+gf_defrag_pattern_match (gf_defrag_info_t *defrag, char *name, uint64_t size)
+{
+        gf_defrag_pattern_list_t *trav = NULL;
+        gf_boolean_t               match = _gf_false;
+        gf_boolean_t               ret = _gf_false;
+
+        GF_VALIDATE_OR_GOTO ("dht", defrag, out);
+
+        trav = defrag->defrag_pattern;
+        while (trav) {
+                if (!fnmatch (trav->path_pattern, name, FNM_NOESCAPE)) {
+                        match = _gf_true;
+                        break;
+                }
+                trav = trav->next;
+        }
+
+        if ((match == _gf_true) && (size >= trav->size))
+                ret = _gf_true;
+
+ out:
+        return ret;
+}
+
+/* We do a depth first traversal of directories. But before we move into
+ * subdirs, we complete the data migration of those directories whose layouts
+ * have been fixed
+ */
+
+int
+gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                        dict_t *migrate_data)
+{
+        int                      ret            = -1;
+        loc_t                    entry_loc      = {0,};
+        fd_t                    *fd             = NULL;
+        gf_dirent_t              entries;
+        gf_dirent_t             *tmp            = NULL;
+        gf_dirent_t             *entry          = NULL;
+        gf_boolean_t             free_entries   = _gf_false;
+        off_t                    offset         = 0;
+        dict_t                  *dict           = NULL;
+        struct iatt              iatt           = {0,};
+        int32_t                  op_errno       = 0;
+        char                    *uuid_str       = NULL;
+        uuid_t                   node_uuid      = {0,};
+        int                      readdir_operrno = 0;
+        struct timeval           dir_start      = {0,};
+        struct timeval           end            = {0,};
+        double                   elapsed        = {0,};
+        struct timeval           start          = {0,};
+        int32_t                  err            = 0;
+
+        gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
+                loc->path);
+        gettimeofday (&dir_start, NULL);
+
+        fd = fd_create (loc->inode, defrag->pid);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
+                goto out;
+        }
+
+        ret = syncop_opendir (this, loc, fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
+                        loc->path);
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&entries.list);
+
+        while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
+                                       &entries)) != 0) {
+
+                if (ret < 0) {
+
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s."
+                                " Aborting migrate-data",
+                                strerror(readdir_operrno));
+                        goto out;
+                }
+
+                /* Need to keep track of ENOENT errno, that means, there is no
+                   need to send more readdirp() */
+                readdir_operrno = errno;
+
+                if (list_empty (&entries.list))
+                        break;
+
+                free_entries = _gf_true;
+
+                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
+                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                                ret = 1;
+                                goto out;
+                        }
+
+                        offset = entry->d_off;
+
+                        if (!strcmp (entry->d_name, ".") ||
+                            !strcmp (entry->d_name, ".."))
+                                continue;
+
+                        if (IA_ISDIR (entry->d_stat.ia_type))
+                                continue;
+
+                        defrag->num_files_lookedup++;
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&start, NULL);
+                        }
+                        if (defrag->defrag_pattern &&
+                            (gf_defrag_pattern_match (defrag, entry->d_name,
+                                                      entry->d_stat.ia_size)
+                             == _gf_false)) {
+                                continue;
+                        }
+                        loc_wipe (&entry_loc);
+                        ret =dht_build_child_loc (this, &entry_loc, loc,
+                                                  entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
+                                        " build failed");
+                                goto out;
+                        }
+
+                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+
+                        if (uuid_is_null (loc->gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        uuid_copy (entry_loc.pargfid, loc->gfid);
+
+                        entry_loc.inode->ia_type = entry->d_stat.ia_type;
+
+                        ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
+                                             NULL, NULL);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s"
+                                        " lookup failed", entry_loc.path);
+                                continue;
+                        }
+
+                        ret = syncop_getxattr (this, &entry_loc, &dict,
+                                               GF_XATTR_NODE_UUID_KEY);
+                        if(ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "get node-uuid for %s", entry_loc.path);
+                                continue;
+                        }
+
+                        ret = dict_get_str (dict, GF_XATTR_NODE_UUID_KEY,
+                                            &uuid_str);
+                        if(ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "get node-uuid from dict for %s",
+                                        entry_loc.path);
+                                continue;
+                        }
+
+                        if (uuid_parse (uuid_str, node_uuid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "uuid_parse "
+                                        "failed for %s", entry_loc.path);
+                                continue;
+                        }
+
+                        /* if file belongs to different node, skip migration
+                         * the other node will take responsibility of migration
+                         */
+                        if (uuid_compare (node_uuid, defrag->node_uuid)) {
+                                gf_log (this->name, GF_LOG_TRACE, "%s does not"
+                                        "belong to this node", entry_loc.path);
+                                continue;
+                        }
+
+                        uuid_str = NULL;
+
+                        dict_del (dict, GF_XATTR_NODE_UUID_KEY);
+
+
+                        /* if distribute is present, it will honor this key.
+                         * -1 is returned if distribute is not present or file
+                         * doesn't have a link-file. If file has link-file, the
+                         * path of link-file will be the value, and also that
+                         * guarantees that file has to be mostly migrated */
+
+                        ret = syncop_getxattr (this, &entry_loc, &dict,
+                                               GF_XATTR_LINKINFO_KEY);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_TRACE, "failed to "
+                                        "get link-to key for %s",
+                                        entry_loc.path);
+                                continue;
+                        }
+
+                        ret = syncop_setxattr (this, &entry_loc, migrate_data,
+                                               0);
+                        if (ret) {
+                                err = op_errno;
+                                /* errno is overloaded. See
+                                 * rebalance_task_completion () */
+                                if (err != ENOSPC) {
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "migrate-data skipped for %s"
+                                                " due to space constraints",
+                                                entry_loc.path);
+                                        defrag->skipped +=1;
+                                } else{
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "migrate-data failed for %s",
+                                                entry_loc.path);
+                                        defrag->total_failures +=1;
+                                }
+                        }
+
+                        if (ret == -1) {
+                                op_errno = errno;
+                                ret = gf_defrag_handle_migrate_error (op_errno,
+                                                                      defrag);
+
+                                if (!ret)
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "migrate-data on %s failed: %s",
+                                                entry_loc.path,
+                                                strerror (op_errno));
+                                else if (ret == 1)
+                                        continue;
+                                else if (ret == -1)
+                                        goto out;
+                        }
+
+                        LOCK (&defrag->lock);
+                        {
+                                defrag->total_files += 1;
+                                defrag->total_data += iatt.ia_size;
+                        }
+                        UNLOCK (&defrag->lock);
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&end, NULL);
+                                elapsed = (end.tv_sec - start.tv_sec) * 1e6 +
+                                          (end.tv_usec - start.tv_usec);
+                                gf_log (this->name, GF_LOG_INFO, "Migration of "
+                                        "file:%s size:%"PRIu64" bytes took %.2f"
+                                        "secs", entry_loc.path, iatt.ia_size,
+                                         elapsed/1e6);
+                        }
+                }
+
+                gf_dirent_free (&entries);
+                free_entries = _gf_false;
+                INIT_LIST_HEAD (&entries.list);
+
+                if (readdir_operrno == ENOENT)
+                        break;
+        }
+
+        gettimeofday (&end, NULL);
+        elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 +
+                  (end.tv_usec - dir_start.tv_usec);
+        gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took "
+                "%.2f secs", loc->path, elapsed/1e6);
+        ret = 0;
+out:
+        if (free_entries)
+                gf_dirent_free (&entries);
+
+        loc_wipe (&entry_loc);
+
+        if (dict)
+                dict_unref(dict);
+
+        if (fd)
+                fd_unref (fd);
+        return ret;
+
+}
+
+
+int
+gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                  dict_t *fix_layout, dict_t *migrate_data)
+{
+        int                      ret            = -1;
+        loc_t                    entry_loc      = {0,};
+        fd_t                    *fd             = NULL;
+        gf_dirent_t              entries;
+        gf_dirent_t             *tmp            = NULL;
+        gf_dirent_t             *entry          = NULL;
+        gf_boolean_t             free_entries   = _gf_false;
+        dict_t                  *dict           = NULL;
+        off_t                    offset         = 0;
+        struct iatt              iatt           = {0,};
+        int                      readdirp_errno = 0;
+
+        ret = syncop_lookup (this, loc, NULL, &iatt, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Lookup failed on %s",
+                        loc->path);
+                goto out;
+        }
+
+        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                ret = gf_defrag_migrate_data (this, defrag, loc, migrate_data);
+                if (ret)
+                        goto out;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "fix layout called on %s", loc->path);
+
+        fd = fd_create (loc->inode, defrag->pid);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_opendir (this, loc, fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
+                        loc->path);
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&entries.list);
+        while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
+                &entries)) != 0)
+        {
+
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s"
+                                ". Aborting fix-layout",strerror(errno));
+                        goto out;
+                }
+
+                /* Need to keep track of ENOENT errno, that means, there is no
+                   need to send more readdirp() */
+                readdirp_errno = errno;
+
+                if (list_empty (&entries.list))
+                        break;
+
+                free_entries = _gf_true;
+
+                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
+                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                                ret = 1;
+                                goto out;
+                        }
+
+                        offset = entry->d_off;
+
+                        if (!strcmp (entry->d_name, ".") ||
+                            !strcmp (entry->d_name, ".."))
+                                continue;
+
+                        if (!IA_ISDIR (entry->d_stat.ia_type))
+                                continue;
+
+                        loc_wipe (&entry_loc);
+                        ret =dht_build_child_loc (this, &entry_loc, loc,
+                                                  entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
+                                        " build failed");
+                                goto out;
+                        }
+
+                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        entry_loc.inode->ia_type = entry->d_stat.ia_type;
+
+                        uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+                        if (uuid_is_null (loc->gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        uuid_copy (entry_loc.pargfid, loc->gfid);
+
+                        ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
+                                             NULL, NULL);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s"
+                                        " lookup failed", entry_loc.path);
+                                continue;
+                        }
+
+                        ret = syncop_setxattr (this, &entry_loc, fix_layout,
+                                               0);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Setxattr "
+                                        "failed for %s", entry_loc.path);
+                                defrag->defrag_status =
+                                GF_DEFRAG_STATUS_FAILED;
+                                defrag->total_failures ++;
+                                goto out;
+                        }
+                        ret = gf_defrag_fix_layout (this, defrag, &entry_loc,
+                                                    fix_layout, migrate_data);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Fix layout "
+                                        "failed for %s", entry_loc.path);
+                                defrag->total_failures++;
+                                goto out;
+                        }
+
+                }
+                gf_dirent_free (&entries);
+                free_entries = _gf_false;
+                INIT_LIST_HEAD (&entries.list);
+                if (readdirp_errno == ENOENT)
+                        break;
+        }
+
+        ret = 0;
+out:
+        if (free_entries)
+                gf_dirent_free (&entries);
+
+        loc_wipe (&entry_loc);
+
+        if (dict)
+                dict_unref(dict);
+
+        if (fd)
+                fd_unref (fd);
+
+        return ret;
+
+}
+
+
+int
+gf_defrag_start_crawl (void *data)
+{
+        xlator_t                *this   = NULL;
+        dht_conf_t              *conf   = NULL;
+        gf_defrag_info_t        *defrag = NULL;
+        int                      ret    = -1;
+        loc_t                    loc    = {0,};
+        struct iatt              iatt   = {0,};
+        struct iatt              parent = {0,};
+        dict_t                  *fix_layout = NULL;
+        dict_t                  *migrate_data = NULL;
+        dict_t                  *status = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+
+        this = data;
+        if (!this)
+                goto out;
+
+        ctx = this->ctx;
+        if (!ctx)
+                goto out;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        defrag = conf->defrag;
+        if (!defrag)
+                goto out;
+
+        gettimeofday (&defrag->start_time, NULL);
+        dht_build_root_inode (this, &defrag->root_inode);
+        if (!defrag->root_inode)
+                goto out;
+
+        dht_build_root_loc (defrag->root_inode, &loc);
+
+        /* fix-layout on '/' first */
+
+        ret = syncop_lookup (this, &loc, NULL, &iatt, NULL, &parent);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "look up on / failed");
+                goto out;
+        }
+
+        fix_layout = dict_new ();
+        if (!fix_layout) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set dict str");
+                goto out;
+        }
+
+        ret = syncop_setxattr (this, &loc, fix_layout, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "fix layout on %s failed",
+                        loc.path);
+                defrag->total_failures++;
+                goto out;
+        }
+
+        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                migrate_data = dict_new ();
+                if (!migrate_data) {
+                        ret = -1;
+                        goto out;
+                }
+                if (defrag->cmd == GF_DEFRAG_CMD_START_FORCE)
+                        ret = dict_set_str (migrate_data,
+                                            "distribute.migrate-data", "force");
+                else
+                        ret = dict_set_str (migrate_data,
+                                            "distribute.migrate-data",
+                                            "non-force");
+                if (ret)
+                        goto out;
+        }
+        ret = gf_defrag_fix_layout (this, defrag, &loc, fix_layout,
+                                    migrate_data);
+        if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
+            (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
+                defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+        }
+
+
+
+out:
+        LOCK (&defrag->lock);
+        {
+                status = dict_new ();
+                gf_defrag_status_get (defrag, status);
+                if (ctx->notify)
+                        ctx->notify (GF_EN_DEFRAG_STATUS, status);
+                if (status)
+                        dict_unref (status);
+                defrag->is_exiting = 1;
+        }
+        UNLOCK (&defrag->lock);
+
+        if (defrag) {
+                GF_FREE (defrag);
+                conf->defrag = NULL;
+        }
+
+        return ret;
+}
+
+
+static int
+gf_defrag_done  (int ret, call_frame_t *sync_frame, void *data)
+{
+        gf_listener_stop (sync_frame->this);
+
+        STACK_DESTROY (sync_frame->root);
+        kill (getpid(), SIGTERM);
+        return 0;
+}
+
+void *
+gf_defrag_start (void *data)
+{
+        int                      ret    = -1;
+        call_frame_t            *frame  = NULL;
+        dht_conf_t              *conf   = NULL;
+        gf_defrag_info_t        *defrag = NULL;
+        xlator_t                *this  = NULL;
+
+        this = data;
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        defrag = conf->defrag;
+        if (!defrag)
+                goto out;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        frame->root->pid = GF_CLIENT_PID_DEFRAG;
+
+        defrag->pid = frame->root->pid;
+
+        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
+
+        ret = synctask_new (this->ctx->env, gf_defrag_start_crawl,
+                            gf_defrag_done, frame, this);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Could not create"
+                        " task for rebalance");
+out:
+        return NULL;
+}
+
+int
+gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
+{
+        int      ret    = 0;
+        uint64_t files  = 0;
+        uint64_t size   = 0;
+        uint64_t lookup = 0;
+        uint64_t failures = 0;
+        uint64_t skipped = 0;
+        char     *status = "";
+        double   elapsed = 0;
+        struct timeval end = {0,};
+
+
+        if (!defrag)
+                goto out;
+
+        ret = 0;
+        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
+                goto out;
+
+        files  = defrag->total_files;
+        size   = defrag->total_data;
+        lookup = defrag->num_files_lookedup;
+        failures = defrag->total_failures;
+        skipped = defrag->skipped;
+
+        gettimeofday (&end, NULL);
+
+        elapsed = end.tv_sec - defrag->start_time.tv_sec;
+
+        if (!dict)
+                goto log;
+
+        ret = dict_set_uint64 (dict, "files", files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set file count");
+
+        ret = dict_set_uint64 (dict, "size", size);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set size of xfer");
+
+        ret = dict_set_uint64 (dict, "lookups", lookup);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set lookedup file count");
+
+
+        ret = dict_set_int32 (dict, "status", defrag->defrag_status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set status");
+        if (elapsed) {
+                ret = dict_set_double (dict, "run-time", elapsed);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set run-time");
+        }
+
+        ret = dict_set_uint64 (dict, "failures", failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set failure count");
+
+        ret = dict_set_uint64 (dict, "skipped", skipped);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set skipped file count");
+log:
+        switch (defrag->defrag_status) {
+        case GF_DEFRAG_STATUS_NOT_STARTED:
+                status = "not started";
+                break;
+        case GF_DEFRAG_STATUS_STARTED:
+                status = "in progress";
+                break;
+        case GF_DEFRAG_STATUS_STOPPED:
+                status = "stopped";
+                break;
+        case GF_DEFRAG_STATUS_COMPLETE:
+                status = "completed";
+                break;
+        case GF_DEFRAG_STATUS_FAILED:
+                status = "failed";
+                break;
+        default:
+                break;
+        }
+
+        gf_log (THIS->name, GF_LOG_INFO, "Rebalance is %s. Time taken is %.2f "
+                "secs", status, elapsed);
+        gf_log (THIS->name, GF_LOG_INFO, "Files migrated: %"PRIu64", size: %"
+                PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: "
+                "%"PRIu64, files, size, lookup, failures, skipped);
+
+
+out:
+        return 0;
+}
+
+int
+gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output)
+{
+        /* TODO: set a variable 'stop_defrag' here, it should be checked
+           in defrag loop */
+        int     ret = -1;
+        GF_ASSERT (defrag);
+
+        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_INFO, "Received stop command on rebalance");
+        defrag->defrag_status = GF_DEFRAG_STATUS_STOPPED;
+
+        if (output)
+                gf_defrag_status_get (defrag, output);
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index e5532f1bc..5d6f4f232 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /* TODO: link(oldpath, newpath) fails if newpath already exists. DHT should
@@ -33,373 +24,735 @@
 
 int
 dht_rename_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, struct stat *stbuf)
+                    int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev = cookie;
+
+        if (op_ret == -1) {
+                /* TODO: undo the damage */
+
+                gf_log (this->name, GF_LOG_INFO,
+                        "rename %s -> %s on %s failed (%s)",
+                        local->loc.path, local->loc2.path,
+                        prev->this->name, strerror (op_errno));
+
+                local->op_ret   = op_ret;
+                local->op_errno = op_errno;
+                goto unwind;
+        }
+        /* TODO: construct proper stbuf for dir */
+        /*
+         * FIXME: is this the correct way to build stbuf and
+         * parent bufs?
+         */
+        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        dht_iatt_merge (this, &local->preoldparent, preoldparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->postoldparent, postoldparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->preparent, prenewparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->postparent, postnewparent,
+                        prev->this);
 
 
-	local = frame->local;
-	prev = cookie;
-
-	if (op_ret == -1) {
-		/* TODO: undo the damage */
-
-		gf_log (this->name, GF_LOG_ERROR,
-			"rename %s -> %s on %s failed (%s)",
-			local->loc.path, local->loc2.path,
-			prev->this->name, strerror (op_errno));
-
-		local->op_ret   = op_ret;
-		local->op_errno = op_errno;
-	} else {
-		/* TODO: construct proper stbuf for dir */
-		local->stbuf = *stbuf;
-	}
+unwind:
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                WIPE (&local->preoldparent);
+                WIPE (&local->postoldparent);
+                WIPE (&local->preparent);
+                WIPE (&local->postparent);
+
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+                DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                                  &local->stbuf, &local->preoldparent,
+                                  &local->postoldparent,
+                                  &local->preparent, &local->postparent, xdata);
+        }
+
+        return 0;
+}
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  &local->stbuf);
-	}
 
-	return 0;
+int
+dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                           struct iatt *preoldparent,
+                           struct iatt *postoldparent,
+                           struct iatt *prenewparent,
+                           struct iatt *postnewparent, dict_t *xdata)
+{
+        dht_conf_t   *conf = NULL;
+        dht_local_t  *local = NULL;
+        int           call_cnt = 0;
+        call_frame_t *prev = NULL;
+        int           i = 0;
+
+        conf = this->private;
+        local = frame->local;
+        prev = cookie;
+
+        if (op_ret == -1) {
+                /* TODO: undo the damage */
+
+                gf_log (this->name, GF_LOG_INFO,
+                        "rename %s -> %s on %s failed (%s)",
+                        local->loc.path, local->loc2.path,
+                        prev->this->name, strerror (op_errno));
+
+                local->op_ret   = op_ret;
+                local->op_errno = op_errno;
+                goto unwind;
+        }
+        /* TODO: construct proper stbuf for dir */
+        /*
+         * FIXME: is this the correct way to build stbuf and
+         * parent bufs?
+         */
+        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        dht_iatt_merge (this, &local->preoldparent, preoldparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->postoldparent, postoldparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->preparent, prenewparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->postparent, postnewparent,
+                        prev->this);
+
+        call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
+
+        if (!local->call_cnt)
+                goto unwind;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == local->dst_hashed)
+                        continue;
+                STACK_WIND (frame, dht_rename_dir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->rename,
+                            &local->loc, &local->loc2, NULL);
+                if (!--call_cnt)
+                        break;
+        }
+
+
+        return 0;
+unwind:
+        WIPE (&local->preoldparent);
+        WIPE (&local->postoldparent);
+        WIPE (&local->preparent);
+        WIPE (&local->postparent);
+
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                          &local->stbuf, &local->preoldparent,
+                          &local->postoldparent,
+                          &local->preparent, &local->postparent, NULL);
+
+        return 0;
 }
 
 
-
 int
 dht_rename_dir_do (call_frame_t *frame, xlator_t *this)
 {
-	dht_local_t  *local = NULL;
-	dht_conf_t   *conf = NULL;
-	int           i = 0;
-
-	conf = this->private;
-	local = frame->local;
+        dht_local_t  *local = NULL;
 
-	if (local->op_ret == -1)
-		goto err;
+        local = frame->local;
 
-	local->call_cnt = conf->subvolume_cnt;
-	local->op_ret = 0;
+        if (local->op_ret == -1)
+                goto err;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_rename_dir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->rename,
-			    &local->loc, &local->loc2);
-	}
+        local->op_ret = 0;
 
-	return 0;
+        STACK_WIND (frame, dht_rename_hashed_dir_cbk,
+                    local->dst_hashed,
+                    local->dst_hashed->fops->rename,
+                    &local->loc, &local->loc2, NULL);
+        return 0;
 
 err:
-	DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	return 0;
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno, NULL, NULL,
+                          NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 
 int
 dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int op_ret, int op_errno, gf_dirent_t *entries)
+                        int op_ret, int op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = -1;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = -1;
+        call_frame_t *prev = NULL;
 
-	local = frame->local;
-	prev  = cookie;
+        local = frame->local;
+        prev  = cookie;
 
-	if (op_ret > 2) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"readdir on %s for %s returned %d entries",
-			prev->this->name, local->loc.path, op_ret);
-		local->op_ret = -1;
-		local->op_errno = ENOTEMPTY;
-	}
+        if (op_ret > 2) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "readdir on %s for %s returned %d entries",
+                        prev->this->name, local->loc.path, op_ret);
+                local->op_ret = -1;
+                local->op_errno = ENOTEMPTY;
+        }
 
-	this_call_cnt = dht_frame_return (frame);
+        this_call_cnt = dht_frame_return (frame);
 
-	if (is_last_call (this_call_cnt)) {
-		dht_rename_dir_do (frame, this);
-	}
+        if (is_last_call (this_call_cnt)) {
+                dht_rename_dir_do (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int op_ret, int op_errno, fd_t *fd)
+                        int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = -1;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = -1;
+        call_frame_t *prev = NULL;
 
 
-	local = frame->local;
-	prev  = cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"opendir on %s for %s failed (%s)",
-			prev->this->name, local->loc.path,
-			strerror (op_errno));
-		goto err;
-	}
+        local = frame->local;
+        prev  = cookie;
 
-	STACK_WIND (frame, dht_rename_readdir_cbk,
-		    prev->this, prev->this->fops->readdir,
-		    local->fd, 4096, 0);
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "opendir on %s for %s failed (%s)",
+                        prev->this->name, local->loc.path,
+                        strerror (op_errno));
+                goto err;
+        }
 
-	return 0;
+        STACK_WIND (frame, dht_rename_readdir_cbk,
+                    prev->this, prev->this->fops->readdir,
+                    local->fd, 4096, 0, NULL);
+
+        return 0;
 
 err:
-	this_call_cnt = dht_frame_return (frame);
+        this_call_cnt = dht_frame_return (frame);
 
-	if (is_last_call (this_call_cnt)) {
-		dht_rename_dir_do (frame, this);
-	}
+        if (is_last_call (this_call_cnt)) {
+                dht_rename_dir_do (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_rename_dir (call_frame_t *frame, xlator_t *this)
 {
-	dht_conf_t  *conf = NULL;
-	dht_local_t *local = NULL;
-	int          i = 0;
-	int          op_errno = -1;
+        dht_conf_t  *conf = NULL;
+        dht_local_t *local = NULL;
+        int          i = 0;
+        int          op_errno = -1;
 
 
-	conf = frame->this->private;
-	local = frame->local;
+        conf = frame->this->private;
+        local = frame->local;
 
-	local->call_cnt = conf->subvolume_cnt;
+        local->call_cnt = conf->subvolume_cnt;
 
-	local->fd = fd_create (local->loc.inode, frame->root->pid);
-	if (!local->fd) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->subvolume_status[i]) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "one of the subvolumes down (%s)",
+                                conf->subvolumes[i]->name);
+                        op_errno = ENOTCONN;
+                        goto err;
+                }
+        }
 
-	local->op_ret = 0;
+        local->fd = fd_create (local->loc.inode, frame->root->pid);
+        if (!local->fd) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	if (!local->dst_cached) {
-		dht_rename_dir_do (frame, this);
-		return 0;
-	}
+        local->op_ret = 0;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_rename_opendir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->opendir,
-			    &local->loc2, local->fd);
-	}
+        if (!local->dst_cached) {
+                dht_rename_dir_do (frame, this);
+                return 0;
+        }
 
-	return 0;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_rename_opendir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->opendir,
+                            &local->loc2, local->fd, NULL);
+        }
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL);
-	return 0;
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL, NULL);
+        return 0;
 }
+#define DHT_MARK_FOP_INTERNAL(xattr) do {                                      \
+                int tmp = -1;                                                  \
+                if (!xattr) {                                                  \
+                        xattr = dict_new ();                                   \
+                        if (!xattr)                                            \
+                                break;                                         \
+                }                                                              \
+                tmp = dict_set_str (xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); \
+                if (tmp) {                                                     \
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set"      \
+                                " internal dict key for %s", local->loc.path); \
+                }                                                              \
+        }while (0)
+int
+dht_rename_done (call_frame_t *frame, xlator_t *this)
+{
+        dht_local_t     *local = NULL;
 
+        local = frame->local;
+
+        if (local->linked == _gf_true) {
+                local->linked = _gf_false;
+                dht_linkfile_attr_heal (frame, this);
+        }
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                          &local->stbuf, &local->preoldparent,
+                          &local->postoldparent, &local->preparent,
+                          &local->postparent, NULL);
+
+        return 0;
+}
 
 int
 dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
-	int           this_call_cnt = 0;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           this_call_cnt = 0;
+
+        local = frame->local;
+        prev  = cookie;
+
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "!local, should not happen");
+                goto out;
+        }
+
+        this_call_cnt = dht_frame_return (frame);
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: unlink on %s failed (%s)",
+                        local->loc.path, prev->this->name, strerror (op_errno));
+        }
+
+        WIPE (&local->preoldparent);
+        WIPE (&local->postoldparent);
+        WIPE (&local->preparent);
+        WIPE (&local->postparent);
+
+        if (is_last_call (this_call_cnt)) {
+                dht_rename_done (frame, this);
+        }
+
+out:
+        return 0;
+}
 
-	local = frame->local;
-	prev  = cookie;
 
-	this_call_cnt = dht_frame_return (frame);
+int
+dht_rename_cleanup (call_frame_t *frame)
+{
+        dht_local_t *local = NULL;
+        xlator_t    *this = NULL;
+        xlator_t    *src_hashed = NULL;
+        xlator_t    *src_cached = NULL;
+        xlator_t    *dst_hashed = NULL;
+        xlator_t    *dst_cached = NULL;
+        int          call_cnt = 0;
+        dict_t      *xattr      = NULL;
+
+        local = frame->local;
+        this  = frame->this;
+
+        src_hashed = local->src_hashed;
+        src_cached = local->src_cached;
+        dst_hashed = local->dst_hashed;
+        dst_cached = local->dst_cached;
+
+        if (src_cached == dst_cached)
+                goto nolinks;
+
+        if (dst_hashed != src_hashed && dst_hashed != src_cached)
+                call_cnt++;
+
+        if (src_cached != dst_hashed)
+                call_cnt++;
+
+        local->call_cnt = call_cnt;
+
+        if (!call_cnt)
+                goto nolinks;
+
+        DHT_MARK_FOP_INTERNAL (xattr);
+
+        if (dst_hashed != src_hashed && dst_hashed != src_cached) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "unlinking linkfile %s @ %s => %s",
+                        local->loc.path, dst_hashed->name, src_cached->name);
+                STACK_WIND (frame, dht_rename_unlink_cbk,
+                            dst_hashed, dst_hashed->fops->unlink,
+                            &local->loc, 0, xattr);
+        }
+
+        if (src_cached != dst_hashed) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "unlinking link %s => %s (%s)", local->loc.path,
+                        local->loc2.path, src_cached->name);
+                STACK_WIND (frame, dht_rename_unlink_cbk,
+                            src_cached, src_cached->fops->unlink,
+                            &local->loc2, 0, xattr);
+        }
+
+        if (xattr)
+                dict_unref (xattr);
+
+        return 0;
 
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"unlink on %s failed (%s)",
-			prev->this->name, strerror (op_errno));
-	}
+nolinks:
+        WIPE (&local->preoldparent);
+        WIPE (&local->postoldparent);
+        WIPE (&local->preparent);
+        WIPE (&local->postparent);
+
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                          &local->stbuf, &local->preoldparent,
+                          &local->postoldparent, &local->preparent,
+                          &local->postparent, NULL);
+
+        return 0;
+}
 
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-				  &local->stbuf);
 
-	return 0;
+int
+dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             inode_t *inode, struct iatt *stbuf,
+                             struct iatt *preparent, struct iatt *postparent,
+                             dict_t *xdata)
+{
+        call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+
+        prev = cookie;
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "link/file %s on %s failed (%s)",
+                        local->loc.path, prev->this->name, strerror (op_errno));
+        }
+
+        if (local->linked == _gf_true) {
+                local->linked = _gf_false;
+                dht_linkfile_attr_heal (frame, this);
+        }
+        DHT_STACK_DESTROY (frame);
+
+        return 0;
 }
 
 
 int
 dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int32_t op_errno, struct stat *stbuf)
+                int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                struct iatt *preoldparent, struct iatt *postoldparent,
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *src_hashed = NULL;
-	xlator_t     *src_cached = NULL;
-	xlator_t     *dst_hashed = NULL;
-	xlator_t     *dst_cached = NULL;
-	xlator_t     *rename_subvol = NULL;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        xlator_t     *src_hashed = NULL;
+        xlator_t     *src_cached = NULL;
+        xlator_t     *dst_hashed = NULL;
+        xlator_t     *dst_cached = NULL;
+        xlator_t     *rename_subvol = NULL;
+        call_frame_t *link_frame = NULL;
+        dht_local_t *link_local = NULL;
+        dict_t       *xattr     = NULL;
+
+        local = frame->local;
+        prev = cookie;
+
+        src_hashed = local->src_hashed;
+        src_cached = local->src_cached;
+        dst_hashed = local->dst_hashed;
+        dst_cached = local->dst_cached;
+
+        if (local->linked == _gf_true)
+                FRAME_SU_UNDO (frame, dht_local_t);
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: rename on %s failed (%s)", local->loc.path,
+                        prev->this->name, strerror (op_errno));
+                local->op_ret   = op_ret;
+                local->op_errno = op_errno;
+                goto cleanup;
+        }
+
+        if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
+                link_frame = copy_frame (frame);
+                if (!link_frame) {
+                        goto err;
+                }
+
+                /* fop value sent as maxvalue because it is not used
+                   anywhere in this case */
+                link_local = dht_local_init (link_frame, &local->loc2, NULL,
+                                             GF_FOP_MAXVALUE);
+                if (!link_local) {
+                        goto err;
+                }
+
+                if (link_local->loc.inode)
+                        inode_unref (link_local->loc.inode);
+                link_local->loc.inode = inode_ref (local->loc.inode);
+                uuid_copy (link_local->gfid, local->loc.inode->gfid);
+
+                dht_linkfile_create (link_frame, dht_rename_links_create_cbk,
+                                     this, src_cached, dst_hashed,
+                                     &link_local->loc);
+        }
 
-	local = frame->local;
-	prev = cookie;
+err:
+        /* Merge attrs only from src_cached. In case there of src_cached !=
+         * dst_hashed, this ignores linkfile attrs. */
+        if (prev->this == src_cached) {
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->preoldparent, preoldparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->postoldparent, postoldparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->preparent, prenewparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->postparent, postnewparent,
+                                prev->this);
+        }
+
+
+        /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
+         *       is called. since rename has already happened on rename_subvol,
+         *       unlink should not be sent for oldpath (either linkfile or cached-file)
+         *       on rename_subvol. */
+        if (src_cached == dst_cached)
+                rename_subvol = src_cached;
+        else
+                rename_subvol = dst_hashed;
+
+        /* TODO: delete files in background */
+
+        if (src_cached != dst_hashed && src_cached != dst_cached)
+                local->call_cnt++;
+
+        if (src_hashed != rename_subvol && src_hashed != src_cached)
+                local->call_cnt++;
+
+        if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
+                local->call_cnt++;
+
+        if (local->call_cnt == 0)
+                goto unwind;
+
+        DHT_MARK_FOP_INTERNAL (xattr);
+
+        if (src_cached != dst_hashed && src_cached != dst_cached) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "deleting old src datafile %s @ %s",
+                        local->loc.path, src_cached->name);
+
+                STACK_WIND (frame, dht_rename_unlink_cbk,
+                            src_cached, src_cached->fops->unlink,
+                            &local->loc, 0, xattr);
+        }
+
+        if (src_hashed != rename_subvol && src_hashed != src_cached) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "deleting old src linkfile %s @ %s",
+                        local->loc.path, src_hashed->name);
+
+                STACK_WIND (frame, dht_rename_unlink_cbk,
+                            src_hashed, src_hashed->fops->unlink,
+                            &local->loc, 0, xattr);
+        }
+
+        if (dst_cached
+            && (dst_cached != dst_hashed)
+            && (dst_cached != src_cached)) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "deleting old dst datafile %s @ %s",
+                        local->loc2.path, dst_cached->name);
+
+                STACK_WIND (frame, dht_rename_unlink_cbk,
+                            dst_cached, dst_cached->fops->unlink,
+                            &local->loc2, 0, xattr);
+        }
+        if (xattr)
+                dict_unref (xattr);
+        return 0;
 
-	src_hashed = local->src_hashed;
-	src_cached = local->src_cached;
-	dst_hashed = local->dst_hashed;
-	dst_cached = local->dst_cached;
+unwind:
+        WIPE (&local->preoldparent);
+        WIPE (&local->postoldparent);
+        WIPE (&local->preparent);
+        WIPE (&local->postparent);
+        if (xattr)
+                dict_unref (xattr);
 
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"rename on %s failed (%s)", prev->this->name,
-			strerror (op_errno));
-		local->op_ret   = op_ret;
-		local->op_errno = op_errno;
-		goto unwind;
-	}
-	
-	/* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
-	 *       is called. since rename has already happened on rename_subvol,
-	 *       unlink should not be sent for oldpath (either linkfile or cached-file)
-	 *       on rename_subvol. */
-	if (src_cached == dst_cached)
-		rename_subvol = src_cached;
-	else
-		rename_subvol = dst_hashed;
+        dht_rename_done (frame, this);
 
-	/* TODO: delete files in background */
+        return 0;
 
-	if (src_cached != dst_hashed && src_cached != dst_cached)
-		local->call_cnt++;
+cleanup:
+        if (xattr)
+                dict_unref (xattr);
+        dht_rename_cleanup (frame);
 
-	if (src_hashed != rename_subvol && src_hashed != src_cached)
-		local->call_cnt++;
+        return 0;
+}
 
-	if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
-		local->call_cnt++;
 
-	if (local->call_cnt == 0)
-		goto unwind;
+int
+dht_do_rename (call_frame_t *frame)
+{
+        dht_local_t *local = NULL;
+        xlator_t    *dst_hashed = NULL;
+        xlator_t    *src_cached = NULL;
+        xlator_t    *dst_cached = NULL;
+        xlator_t    *this = NULL;
+        xlator_t    *rename_subvol = NULL;
 
-	if (src_cached != dst_hashed && src_cached != dst_cached) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"deleting old src datafile %s @ %s",
-			local->loc.path, src_cached->name);
 
-		STACK_WIND (frame, dht_rename_unlink_cbk,
-			    src_cached, src_cached->fops->unlink,
-			    &local->loc);
-	}
+        local = frame->local;
+        this  = frame->this;
 
-	if (src_hashed != rename_subvol && src_hashed != src_cached) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"deleting old src linkfile %s @ %s",
-			local->loc.path, src_hashed->name);
+        dst_hashed = local->dst_hashed;
+        dst_cached = local->dst_cached;
+        src_cached = local->src_cached;
 
-		STACK_WIND (frame, dht_rename_unlink_cbk,
-			    src_hashed, src_hashed->fops->unlink,
-			    &local->loc);
-	}
-
-	if (dst_cached
-	    && (dst_cached != dst_hashed)
-	    && (dst_cached != src_cached)) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"deleting old dst datafile %s @ %s",
-			local->loc2.path, dst_cached->name);
+        if (src_cached == dst_cached)
+                rename_subvol = src_cached;
+        else
+                rename_subvol = dst_hashed;
 
-		STACK_WIND (frame, dht_rename_unlink_cbk,
-			    dst_cached, dst_cached->fops->unlink,
-			    &local->loc2);
-	}
-	return 0;
+        gf_log (this->name, GF_LOG_TRACE,
+                "renaming %s => %s (%s)",
+                local->loc.path, local->loc2.path, rename_subvol->name);
 
-unwind:
-	DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			  &local->stbuf);
+        if (local->linked == _gf_true)
+                FRAME_SU_DO (frame, dht_local_t);
+        STACK_WIND (frame, dht_rename_cbk,
+                    rename_subvol, rename_subvol->fops->rename,
+                    &local->loc, &local->loc2, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_do_rename (call_frame_t *frame)
+dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      inode_t *inode, struct iatt *stbuf,
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
 {
-	dht_local_t *local = NULL;
-	xlator_t    *dst_hashed = NULL;
-	xlator_t    *src_cached = NULL;
-	xlator_t    *dst_cached = NULL;
-	xlator_t    *this = NULL;
-	xlator_t    *rename_subvol = NULL;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           this_call_cnt  = 0;
 
 
-	local = frame->local;
-	this  = frame->this;
+        local = frame->local;
+        prev = cookie;
 
-	dst_hashed = local->dst_hashed;
-	dst_cached = local->dst_cached;
-	src_cached = local->src_cached;
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "link/file on %s failed (%s)",
+                        prev->this->name, strerror (op_errno));
+                local->op_ret   = -1;
+                if (op_errno != ENOENT)
+                        local->op_errno = op_errno;
+        } else if (local->src_cached == prev->this) {
+                /* merge of attr returned only from linkfile creation */
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        }
 
-	if (src_cached == dst_cached)
-		rename_subvol = src_cached;
-	else
-		rename_subvol = dst_hashed;
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                if (local->op_ret == -1)
+                        goto cleanup;
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"renaming %s => %s (%s)",
-		local->loc.path, local->loc2.path, rename_subvol->name);
+                dht_do_rename (frame);
+        }
 
-	STACK_WIND (frame, dht_rename_cbk,
-		    rename_subvol, rename_subvol->fops->rename,
-		    &local->loc, &local->loc2);
+        return 0;
 
-	return 0;
+cleanup:
+        dht_rename_cleanup (frame);
+
+        return 0;
 }
 
 
 int
-dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno,
-		      inode_t *inode, struct stat *stbuf)
+dht_rename_unlink_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *preparent, struct iatt *postparent,
+                             dict_t *xdata)
 {
 	dht_local_t  *local = NULL;
 	call_frame_t *prev = NULL;
-	int           this_call_cnt  = 0;
 
 
 	local = frame->local;
 	prev = cookie;
-	
-	if (op_ret == -1) {
+
+	if ((op_ret == -1) && (op_errno != ENOENT)) {
 		gf_log (this->name, GF_LOG_DEBUG,
-			"link/file on %s failed (%s)",
-			prev->this->name, strerror (op_errno));
+			"unlink of %s on %s failed (%s)",
+			local->loc2.path, prev->this->name,
+                        strerror (op_errno));
 		local->op_ret   = -1;
 		local->op_errno = op_errno;
 	}
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		if (local->op_ret == -1)
-			goto unwind;
-		
-		dht_do_rename (frame);
-	}
+        if (local->op_ret == -1)
+                goto cleanup;
+
+        dht_do_rename (frame);
 
 	return 0;
 
-unwind:
-	DHT_STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			  &local->stbuf);
+cleanup:
+        dht_rename_cleanup (frame);
 
 	return 0;
 }
@@ -408,155 +761,164 @@ unwind:
 int
 dht_rename_create_links (call_frame_t *frame)
 {
-	dht_local_t *local = NULL;
-	xlator_t    *this = NULL;
-	xlator_t    *src_hashed = NULL;
-	xlator_t    *src_cached = NULL;
-	xlator_t    *dst_hashed = NULL;
-	xlator_t    *dst_cached = NULL;
-	int          call_cnt = 0;
-
-
-	local = frame->local;
-	this  = frame->this;
-
-	src_hashed = local->src_hashed;
-	src_cached = local->src_cached;
-	dst_hashed = local->dst_hashed;
-	dst_cached = local->dst_cached;
-
-	if (src_cached == dst_cached)
-		goto nolinks;
-
-	if (dst_hashed != src_hashed && dst_hashed != src_cached)
-		call_cnt++;
-
-	if (src_cached != dst_hashed)
-		call_cnt++;
-
-	local->call_cnt = call_cnt;
-
-	if (dst_hashed != src_hashed && dst_hashed != src_cached) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"linkfile %s @ %s => %s",
-			local->loc.path, dst_hashed->name, src_cached->name);
-		dht_linkfile_create (frame, dht_rename_links_cbk,
+        dht_local_t *local = NULL;
+        xlator_t    *this = NULL;
+        xlator_t    *src_hashed = NULL;
+        xlator_t    *src_cached = NULL;
+        xlator_t    *dst_hashed = NULL;
+        xlator_t    *dst_cached = NULL;
+        int          call_cnt = 0;
+        dict_t      *xattr = NULL;
+
+
+        local = frame->local;
+        this  = frame->this;
+
+        src_hashed = local->src_hashed;
+        src_cached = local->src_cached;
+        dst_hashed = local->dst_hashed;
+        dst_cached = local->dst_cached;
+
+        DHT_MARK_FOP_INTERNAL (xattr);
+
+        if (src_cached == dst_cached) {
+                if (dst_hashed == dst_cached)
+                        goto nolinks;
+
+		gf_log (this->name, GF_LOG_TRACE,
+			"unlinking dst linkfile %s @ %s",
+			local->loc2.path, dst_hashed->name);
+
+		STACK_WIND (frame, dht_rename_unlink_links_cbk,
+			    dst_hashed, dst_hashed->fops->unlink,
+			    &local->loc2, 0, xattr);
+                return 0;
+        }
+
+        if (dst_hashed != src_hashed && dst_hashed != src_cached)
+                call_cnt++;
+
+        if (src_cached != dst_hashed)
+                call_cnt++;
+
+        local->call_cnt = call_cnt;
+
+        if (dst_hashed != src_hashed && dst_hashed != src_cached) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "linkfile %s @ %s => %s",
+                        local->loc.path, dst_hashed->name, src_cached->name);
+                memcpy (local->gfid, local->loc.inode->gfid, 16);
+		dht_linkfile_create (frame, dht_rename_links_cbk, this,
 				     src_cached, dst_hashed, &local->loc);
 	}
 
 	if (src_cached != dst_hashed) {
-		gf_log (this->name, GF_LOG_DEBUG,
+		gf_log (this->name, GF_LOG_TRACE,
 			"link %s => %s (%s)", local->loc.path,
 			local->loc2.path, src_cached->name);
 		STACK_WIND (frame, dht_rename_links_cbk,
 			    src_cached, src_cached->fops->link,
-			    &local->loc, &local->loc2);
+			    &local->loc, &local->loc2, xattr);
 	}
 
 nolinks:
-	if (!call_cnt) {
-		/* skip to next step */
-		dht_do_rename (frame);
-	}
-
-	return 0;
+        if (!call_cnt) {
+                /* skip to next step */
+                dht_do_rename (frame);
+        }
+        if (xattr)
+                dict_unref (xattr);
+
+        return 0;
 }
 
 
 int
 dht_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc)
+            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-	xlator_t    *src_cached = NULL;
-	xlator_t    *src_hashed = NULL;
-	xlator_t    *dst_cached = NULL;
-	xlator_t    *dst_hashed = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
-	dht_local_t *local = NULL;
-
-
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (oldloc, err);
-	VALIDATE_OR_GOTO (newloc, err);
-
-	src_hashed = dht_subvol_get_hashed (this, oldloc);
-	if (!src_hashed) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			oldloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	src_cached = dht_subvol_get_cached (this, oldloc->inode);
-	if (!src_cached) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no cached subvolume for path=%s", oldloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	dst_hashed = dht_subvol_get_hashed (this, newloc);
-	if (!dst_hashed) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			newloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	if (newloc->inode)
-		dst_cached = dht_subvol_get_cached (this, newloc->inode);
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-	ret = loc_copy (&local->loc, oldloc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-	ret = loc_copy (&local->loc2, newloc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-	local->src_hashed = src_hashed;
-	local->src_cached = src_cached;
-	local->dst_hashed = dst_hashed;
-	local->dst_cached = dst_cached;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"renaming %s (hash=%s/cache=%s) => %s (hash=%s/cache=%s)",
-		oldloc->path, src_hashed->name, src_cached->name,
-		newloc->path, dst_hashed->name,
-		dst_cached ? dst_cached->name : "<nul>");
-
-	if (S_ISDIR (oldloc->inode->st_mode)) {
-		dht_rename_dir (frame, this);
-	} else {
-		local->op_ret = 0;
-		dht_rename_create_links (frame);
-	}
-
-	return 0;
+        xlator_t    *src_cached = NULL;
+        xlator_t    *src_hashed = NULL;
+        xlator_t    *dst_cached = NULL;
+        xlator_t    *dst_hashed = NULL;
+        int          op_errno = -1;
+        int          ret = -1;
+        dht_local_t *local = NULL;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (oldloc, err);
+        VALIDATE_OR_GOTO (newloc, err);
+
+        src_hashed = dht_subvol_get_hashed (this, oldloc);
+        if (!src_hashed) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no subvolume in layout for path=%s",
+                        oldloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        src_cached = dht_subvol_get_cached (this, oldloc->inode);
+        if (!src_cached) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no cached subvolume for path=%s", oldloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        dst_hashed = dht_subvol_get_hashed (this, newloc);
+        if (!dst_hashed) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no subvolume in layout for path=%s",
+                        newloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (newloc->inode)
+                dst_cached = dht_subvol_get_cached (this, newloc->inode);
+
+        local = dht_local_init (frame, oldloc, NULL, GF_FOP_RENAME);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        /* cached_subvol will be set from dht_local_init, reset it to NULL,
+           as the logic of handling rename is different  */
+        local->cached_subvol = NULL;
+
+        ret = loc_copy (&local->loc2, newloc);
+        if (ret == -1) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->src_hashed = src_hashed;
+        local->src_cached = src_cached;
+        local->dst_hashed = dst_hashed;
+        local->dst_cached = dst_cached;
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "renaming %s (hash=%s/cache=%s) => %s (hash=%s/cache=%s)",
+                oldloc->path, src_hashed->name, src_cached->name,
+                newloc->path, dst_hashed->name,
+                dst_cached ? dst_cached->name : "<nul>");
+
+        if (IA_ISDIR (oldloc->inode->ia_type)) {
+                dht_rename_dir (frame, this);
+        } else {
+                local->op_ret = 0;
+                dht_rename_create_links (frame);
+        }
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL, NULL);
 
-	return 0;
+        return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index 45c9586cf..3fe96b1c7 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -26,465 +17,1013 @@
 #include "glusterfs.h"
 #include "xlator.h"
 #include "dht-common.h"
+#include "glusterfs-acl.h"
+
+#define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,cnt,path)    do {       \
+                layout->list[i].start = srt;                            \
+                layout->list[i].stop  = srt + chunk - 1;                \
+                                                                        \
+                gf_log (this->name, GF_LOG_TRACE,                       \
+                        "gave fix: %u - %u on %s for %s",               \
+                        layout->list[i].start, layout->list[i].stop,    \
+                        layout->list[i].xlator->name, path);            \
+        } while (0)
+
+#define DHT_RESET_LAYOUT_RANGE(layout)    do {                          \
+                int cnt = 0;                                            \
+                for (cnt = 0; cnt < layout->cnt; cnt++ ) {              \
+                        layout->list[cnt].start = 0;                    \
+                        layout->list[cnt].stop  = 0;                    \
+                }                                                       \
+        } while (0)
+
+static uint32_t
+dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n)
+{
+	if (o >= old->cnt || n >= new->cnt)
+		return 0;
 
+	if (old->list[o].err > 0 || new->list[n].err > 0)
+		return 0;
 
-int
-dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
-{
-	dht_local_t  *local = NULL;
+        if (old->list[o].start == old->list[o].stop) {
+                return 0;
+        }
 
+        if (new->list[n].start == new->list[n].stop) {
+                return 0;
+        }
 
-	local = frame->local;
-	local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
-				 local->op_errno);
+	if ((old->list[o].start > new->list[n].stop) ||
+	    (old->list[o].stop < new->list[n].start))
+		return 0;
 
-	return 0;
+	return min (old->list[o].stop, new->list[n].stop) -
+	        max (old->list[o].start, new->list[n].start) + 1;
 }
 
 
 int
-dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			    int op_ret, int op_errno)
+dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *subvol = NULL;
-	int           i = 0;
-	dht_layout_t *layout = NULL;
-	int           err = 0;
-	int           this_call_cnt = 0;
-
-	local = frame->local;
-	layout = local->selfheal.layout;
-	prev = cookie;
-	subvol = prev->this;
-
-	if (op_ret == 0)
-		err = 0;
-	else
-		err = op_errno;
-
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].xlator == subvol) {
-			layout->list[i].err = err;
-			break;
-		}
-	}
+        dht_local_t  *local = NULL;
 
-	this_call_cnt = dht_frame_return (frame);
+        local = frame->local;
+        local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
+                                 local->op_errno, NULL);
 
-	if (is_last_call (this_call_cnt)) {
-		dht_selfheal_dir_finish (frame, this, 0);
-	}
-
-	return 0;
+        return 0;
 }
 
 
 int
-dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
-				  dht_layout_t *layout, int i)
+dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno, dict_t *xdata)
 {
-	xlator_t          *subvol = NULL;
-	dict_t            *xattr = NULL;
-	int                ret = 0;
-	xlator_t          *this = NULL;
-	int32_t           *disk_layout = NULL;
-
-
-	subvol = layout->list[i].xlator;
-	this = frame->this;
-
-	xattr = get_new_dict ();
-	if (!xattr) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-	ret = dht_disk_layout_extract (this, layout, i, &disk_layout);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"failed to extract disk layout");
-		goto err;
-	}
-
-	ret = dict_set_bin (xattr, "trusted.glusterfs.dht",
-			    disk_layout, 4 * 4);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"failed to set xattr dictionary");
-		goto err;
-	}
-	disk_layout = NULL;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"setting hash range %u - %u (type %d) on subvolume %s for %s",
-		layout->list[i].start, layout->list[i].stop,
-		layout->type, subvol->name, loc->path);
-
-	dict_ref (xattr);
-
-	STACK_WIND (frame, dht_selfheal_dir_xattr_cbk,
-		    subvol, subvol->fops->setxattr,
-		    loc, xattr, 0);
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        xlator_t     *subvol = NULL;
+        int           i = 0;
+        dht_layout_t *layout = NULL;
+        int           err = 0;
+        int           this_call_cnt = 0;
+
+        local = frame->local;
+        layout = local->selfheal.layout;
+        prev = cookie;
+        subvol = prev->this;
+
+        if (op_ret == 0)
+                err = 0;
+        else
+                err = op_errno;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].xlator == subvol) {
+                        layout->list[i].err = err;
+                        break;
+                }
+        }
+
+        this_call_cnt = dht_frame_return (frame);
+
+        if (is_last_call (this_call_cnt)) {
+                dht_selfheal_dir_finish (frame, this, 0);
+        }
+
+        return 0;
+}
 
-	dict_unref (xattr);
 
-	return 0;
+int
+dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *layout, int i,
+                                  xlator_t *req_subvol)
+{
+        xlator_t          *subvol = NULL;
+        dict_t            *xattr = NULL;
+        int                ret = 0;
+        xlator_t          *this = NULL;
+        int32_t           *disk_layout = NULL;
+        dht_local_t       *local = NULL;
+        dht_conf_t        *conf = NULL;
+
+        local = frame->local;
+        if (req_subvol)
+                subvol = req_subvol;
+        else
+                subvol = layout->list[i].xlator;
+        this = frame->this;
+
+        GF_VALIDATE_OR_GOTO ("", this, err);
+        GF_VALIDATE_OR_GOTO (this->name, layout, err);
+        GF_VALIDATE_OR_GOTO (this->name, local, err);
+        GF_VALIDATE_OR_GOTO (this->name, subvol, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
+
+        xattr = get_new_dict ();
+        if (!xattr) {
+                goto err;
+        }
+
+        ret = dht_disk_layout_extract (this, layout, i, &disk_layout);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: (subvol %s) failed to extract disk layout",
+                        loc->path, subvol->name);
+                goto err;
+        }
+
+        ret = dict_set_bin (xattr, conf->xattr_name, disk_layout, 4 * 4);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: (subvol %s) failed to set xattr dictionary",
+                        loc->path, subvol->name);
+                goto err;
+        }
+        disk_layout = NULL;
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "setting hash range %u - %u (type %d) on subvolume %s for %s",
+                layout->list[i].start, layout->list[i].stop,
+                layout->type, subvol->name, loc->path);
+
+        dict_ref (xattr);
+
+        if (!uuid_is_null (local->gfid))
+                uuid_copy (loc->gfid, local->gfid);
+
+        STACK_WIND (frame, dht_selfheal_dir_xattr_cbk,
+                    subvol, subvol->fops->setxattr,
+                    loc, xattr, 0, NULL);
+
+        dict_unref (xattr);
+
+        return 0;
 
 err:
-	if (xattr)
-		dict_destroy (xattr);
+        if (xattr)
+                dict_destroy (xattr);
 
-	if (disk_layout)
-		FREE (disk_layout);
+        GF_FREE (disk_layout);
 
-	dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
-				    -1, ENOMEM);
-	return 0;
+        dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
+                                    -1, ENOMEM, NULL);
+        return 0;
 }
 
+int
+dht_fix_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+{
+        dht_local_t *local = NULL;
+        int          i = 0;
+        int          count = 0;
+        xlator_t    *this = NULL;
+        dht_conf_t  *conf = NULL;
+        dht_layout_t *dummy = NULL;
+
+        local = frame->local;
+        this = frame->this;
+        conf = this->private;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "writing the new range for all subvolumes");
+
+        local->call_cnt = count = conf->subvolume_cnt;
+
+        for (i = 0; i < layout->cnt; i++) {
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
+
+                if (--count == 0)
+                        goto out;
+        }
+        /* if we are here, subvolcount > layout_count. subvols-per-directory
+         * option might be set here. We need to clear out layout from the
+         * non-participating subvolumes, else it will result in overlaps */
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                        if (--count == 0)
+                                break;
+                }
+        }
+
+        dht_layout_unref (this, dummy);
+out:
+        return 0;
+}
 
 int
 dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-	dht_local_t *local = NULL;
-	int          missing_xattr = 0;
-	int          i = 0;
-	int          ret = 0;
-	xlator_t    *this = NULL;
-
-	local = frame->local;
-	this = frame->this;
-
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err != -1 || !layout->list[i].stop) {
-			/* err != -1 would mean xattr present on the directory
-			 * or the directory is itself non existant.
-			 * !layout->list[i].stop would mean layout absent
-			 */
-			continue;
-		}
-		missing_xattr++;
-	}
+        dht_local_t *local = NULL;
+        int          missing_xattr = 0;
+        int          i = 0;
+        xlator_t    *this = NULL;
+        dht_conf_t   *conf = NULL;
+        dht_layout_t *dummy = NULL;
+
+        local = frame->local;
+        this = frame->this;
+        conf = this->private;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err != -1 || !layout->list[i].stop) {
+                        /* err != -1 would mean xattr present on the directory
+                         * or the directory is non existent.
+                         * !layout->list[i].stop would mean layout absent
+                         */
+
+                        continue;
+                }
+                missing_xattr++;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "%d subvolumes missing xattr for %s",
+                missing_xattr, loc->path);
+
+        if (missing_xattr == 0) {
+                dht_selfheal_dir_finish (frame, this, 0);
+                return 0;
+        }
+
+        local->call_cnt = missing_xattr;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err != -1 || !layout->list[i].stop)
+                        continue;
+
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
+
+                if (--missing_xattr == 0)
+                        break;
+        }
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                }
+        }
+        dht_layout_unref (this, dummy);
+out:
+        return 0;
+}
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"%d subvolumes missing xattr for %s",
-		missing_xattr, loc->path);
+int
+dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, struct iatt *statpre,
+                              struct iatt *statpost, dict_t *xdata)
+{
+        dht_local_t   *local = NULL;
+        dht_layout_t  *layout = NULL;
+        int            this_call_cnt = 0;
 
-	if (missing_xattr == 0) {
-		dht_selfheal_dir_finish (frame, this, 0);
-		return 0;
-	}
+        local  = frame->local;
+        layout = local->selfheal.layout;
 
-	local->call_cnt = missing_xattr;
+        this_call_cnt = dht_frame_return (frame);
 
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err != -1 || !layout->list[i].stop)
-			continue;
+        if (is_last_call (this_call_cnt)) {
+                dht_selfheal_dir_xattr (frame, &local->loc, layout);
+        }
 
-		ret = dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
-
-		if (--missing_xattr == 0)
-			break;
-	}
-	return 0;
+        return 0;
 }
 
 
 int
-dht_selfheal_dir_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			    int op_ret, int op_errno,
-			    inode_t *inode, struct stat *stbuf)
+dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                          int32_t valid, dht_layout_t *layout)
 {
-	dht_local_t   *local = NULL;
-	dht_layout_t  *layout = NULL;
-	call_frame_t  *prev = NULL;
-	xlator_t      *subvol = NULL;
-	int            i = 0;
-	int            this_call_cnt = 0;
-
-
-	local  = frame->local;
-	layout = local->selfheal.layout;
-	prev   = cookie;
-	subvol = prev->this;
-
-	if ((op_ret == 0) || (op_errno == EEXIST)) {
-		for (i = 0; i < layout->cnt; i++) {
-			if (layout->list[i].xlator == subvol) {
-				layout->list[i].err = -1;
-				break;
-			}
-		}
-	}
+        int           missing_attr = 0;
+        int           i     = 0;
+        dht_local_t  *local = NULL;
+        xlator_t     *this = NULL;
+
+        local = frame->local;
+        this = frame->this;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == -1)
+                        missing_attr++;
+        }
+
+        if (missing_attr == 0) {
+                dht_selfheal_dir_xattr (frame, loc, layout);
+                return 0;
+        }
+
+        if (!uuid_is_null (local->gfid))
+                uuid_copy (loc->gfid, local->gfid);
+
+        local->call_cnt = missing_attr;
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == -1) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "setattr for %s on subvol %s",
+                                loc->path, layout->list[i].xlator->name);
+
+                        STACK_WIND (frame, dht_selfheal_dir_setattr_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->setattr,
+                                    loc, stbuf, valid, NULL);
+                }
+        }
+
+        return 0;
+}
 
-	this_call_cnt = dht_frame_return (frame);
+int
+dht_selfheal_dir_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno,
+                            inode_t *inode, struct iatt *stbuf,
+                            struct iatt *preparent, struct iatt *postparent,
+                            dict_t *xdata)
+{
+        dht_local_t   *local = NULL;
+        dht_layout_t  *layout = NULL;
+        call_frame_t  *prev = NULL;
+        xlator_t      *subvol = NULL;
+        int            i = 0;
+        int            this_call_cnt = 0;
+
+
+        local  = frame->local;
+        layout = local->selfheal.layout;
+        prev   = cookie;
+        subvol = prev->this;
+
+        if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
+                for (i = 0; i < layout->cnt; i++) {
+                        if (layout->list[i].xlator == subvol) {
+                                layout->list[i].err = -1;
+                                break;
+                        }
+                }
+        }
+
+        if (op_ret) {
+                gf_log (this->name, ((op_errno == EEXIST) ? GF_LOG_DEBUG :
+                                     GF_LOG_WARNING),
+                        "selfhealing directory %s failed: %s",
+                        local->loc.path, strerror (op_errno));
+                goto out;
+        }
+
+        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        dht_iatt_merge (this, &local->preparent, preparent, prev->this);
+        dht_iatt_merge (this, &local->postparent, postparent, prev->this);
+
+out:
+        this_call_cnt = dht_frame_return (frame);
+
+        if (is_last_call (this_call_cnt)) {
+                dht_selfheal_dir_setattr (frame, &local->loc, &local->stbuf, 0xffffff, layout);
+        }
+
+        return 0;
+}
 
-	if (is_last_call (this_call_cnt)) {
-		dht_selfheal_dir_xattr (frame, &local->loc, layout);
-	}
+void
+dht_selfheal_dir_mkdir_setacl (dict_t *xattr, dict_t *dict)
+{
+        data_t          *acl_default = NULL;
+        data_t          *acl_access = NULL;
+        xlator_t        *this = NULL;
+        int     ret = -1;
+
+        GF_ASSERT (xattr);
+        GF_ASSERT (dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        acl_default = dict_get (xattr, POSIX_ACL_DEFAULT_XATTR);
+
+        if (!acl_default) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "ACL_DEFAULT xattr not present");
+                goto cont;
+        }
+        ret = dict_set (dict, POSIX_ACL_DEFAULT_XATTR, acl_default);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not set ACL_DEFAULT xattr");
+cont:
+        acl_access = dict_get (xattr, POSIX_ACL_ACCESS_XATTR);
+        if (!acl_access) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "ACL_ACCESS xattr not present");
+                goto out;
+        }
+        ret = dict_set (dict, POSIX_ACL_ACCESS_XATTR, acl_access);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not set ACL_ACCESS xattr");
+
+out:
+        return;
+}
 
-	return 0;
+int
+dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
+                        dht_layout_t *layout, int force)
+{
+        int           missing_dirs = 0;
+        int           i     = 0;
+        int           ret   = -1;
+        dht_local_t  *local = NULL;
+        xlator_t     *this = NULL;
+        dict_t       *dict = NULL;
+
+        local = frame->local;
+        this = frame->this;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == ENOENT || force)
+                        missing_dirs++;
+        }
+
+        if (missing_dirs == 0) {
+                dht_selfheal_dir_setattr (frame, loc, &local->stbuf, 0xffffffff, layout);
+                return 0;
+        }
+
+        local->call_cnt = missing_dirs;
+        if (!uuid_is_null (local->gfid)) {
+                dict = dict_new ();
+                if (!dict)
+                        return -1;
+
+                ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set gfid in dict", loc->path);
+        } else if (local->params) {
+                /* Send the dictionary from higher layers directly */
+                dict = dict_ref (local->params);
+        }
+        /* Set acls */
+        if (local->xattr && dict)
+                dht_selfheal_dir_mkdir_setacl (local->xattr, dict);
+
+        if (!dict)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict is NULL, need to make sure gfids are same");
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == ENOENT || force) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "creating directory %s on subvol %s",
+                                loc->path, layout->list[i].xlator->name);
+
+                        STACK_WIND (frame, dht_selfheal_dir_mkdir_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->mkdir,
+                                    loc,
+                                    st_mode_from_ia (local->stbuf.ia_prot,
+                                                     local->stbuf.ia_type),
+                                    0, dict);
+                }
+        }
+
+        if (dict)
+                dict_unref (dict);
+
+        return 0;
 }
 
 
 int
-dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
-			dht_layout_t *layout, int force)
+dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
+                                 dht_layout_t *layout)
 {
-	int           missing_dirs = 0;
-	int           i = 0;
-	dht_local_t  *local = NULL;
-	xlator_t     *this = NULL;
+        int           start = 0;
+        uint32_t      hashval = 0;
+        int           ret = 0;
 
+        ret = dht_hash_compute (this, layout->type, loc->path, &hashval);
+        if (ret == 0) {
+                start = (hashval % layout->cnt);
+        }
 
-	local = frame->local;
-	this = frame->this;
+        return start;
+}
 
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err == ENOENT || force)
-			missing_dirs++;
-	}
+static inline int
+dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
+{
+        int i = 0;
+        int j = 0;
+        int err = 0;
+        int count = 0;
+        dht_conf_t *conf = NULL;
+
+        /* Gets in use only for replace-brick, remove-brick */
+        conf = this->private;
+        for (i = 0; i < layout->cnt; i++) {
+                for (j = 0; j < conf->subvolume_cnt; j++) {
+                        if (conf->decommissioned_bricks[j] &&
+                            conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                                layout->list[i].err = EINVAL;
+                                break;
+                        }
+                }
+        }
+
+        for (i = 0; i < layout->cnt; i++) {
+                err = layout->list[i].err;
+                if (err == -1 || err == 0 || err == ENOENT) {
+			/* Setting list[i].err = -1 is an indication for
+			   dht_selfheal_layout_new_directory() to assign
+			   a range. We set it to -1 based on any one of
+			   the three criteria:
+
+			   - err == -1 already, which means directory
+			     existed but layout was not set on it.
+
+			   - err == 0, which means directory exists and
+			     has an old layout piece which will be
+			     overwritten now.
+
+			   - err == ENOENT, which means directory does
+			     not exist (possibly racing with mkdir or
+			     finishing half done mkdir). The missing
+			     directory will be attempted to be recreated.
+
+			     It is important to note that it is safe
+			     to race with mkdir() as self-heal and
+			     mkdir are idempotent operations. Both will
+			     strive to set the directory and layouts to
+			     the same final state.
+			*/
+                        count++;
+			if (!err)
+				layout->list[i].err = -1;
+                }
+        }
+
+        /* no subvolume has enough space, but can't stop directory creation */
+        if (!count || !new_layout) {
+                for (i = 0; i < layout->cnt; i++) {
+                        err = layout->list[i].err;
+                        if (err == ENOSPC) {
+                                layout->list[i].err = -1;
+                                count++;
+                        }
+                }
+        }
+
+        /* if layout->spread_cnt is set, check if it is <= available
+         * subvolumes (down brick and decommissioned bricks are considered
+         * un-availbale). Else return count (available up bricks) */
+        count = ((layout->spread_cnt &&
+                 (layout->spread_cnt <= count)) ?
+                 layout->spread_cnt : ((count) ? count : 1));
+
+        return count;
+}
 
-	if (missing_dirs == 0) {
-		dht_selfheal_dir_xattr (frame, loc, layout);
-		return 0;
-	}
 
-	local->call_cnt = missing_dirs;
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err == ENOENT || force) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"creating directory %s on subvol %s",
-				loc->path, layout->list[i].xlator->name);
-
-			STACK_WIND (frame, dht_selfheal_dir_mkdir_cbk,
-				    layout->list[i].xlator,
-				    layout->list[i].xlator->fops->mkdir,
-				    loc, local->stbuf.st_mode);
-		}
-	}
+void dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
+					dht_layout_t *new_layout);
 
-	return 0;
-}
+void dht_layout_entry_swap (dht_layout_t *layout, int i, int j);
+void dht_layout_range_swap (dht_layout_t *layout, int i, int j);
+
+/*
+ * It's a bit icky using local variables in a macro, but it makes the rest
+ * of the code a lot clearer.
+ */
+#define OV_ENTRY(x,y)      table[x*new->cnt+y]
 
 void
-dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
-				   dht_layout_t *layout)
+dht_selfheal_layout_maximize_overlap (call_frame_t *frame, loc_t *loc,
+				      dht_layout_t *new, dht_layout_t *old)
 {
-	dht_conf_t  *conf = NULL;
-	xlator_t    *this = NULL;
-	uint32_t     chunk = 0;
-	int          i = 0;
-	uint32_t     start = 0;
-	int          cnt = 0;
-	int          err = 0;
-
-	this = frame->this;
-	conf = this->private;
-
-	for (i = 0; i < layout->cnt; i++) {
-		err = layout->list[i].err;
-		if (err == -1 || err == 0) {
-			layout->list[i].err = -1;
-			cnt++;
-		}
+	int           i            = 0;
+	int           j            = 0;
+	uint32_t      curr_overlap = 0;
+	uint32_t      max_overlap  = 0;
+	int           max_overlap_idx = -1;
+	uint32_t      overlap      = 0;
+        uint32_t     *table = NULL;
+
+	dht_layout_sort_volname (old);
+	/* Now both old_layout->list[] and new_layout->list[]
+	   are match the same xlators/subvolumes. i.e,
+	   old_layout->[i] and new_layout->[i] are referring
+	   to the same subvolumes
+	*/
+
+        /* Build a table of overlaps between new[i] and old[j]. */
+        table = alloca(sizeof(overlap)*old->cnt*new->cnt);
+        if (!table) {
+                return;
+        }
+        memset(table,0,sizeof(overlap)*old->cnt*new->cnt);
+        for (i = 0; i < new->cnt; ++i) {
+                for (j = 0; j < old->cnt; ++j) {
+                        OV_ENTRY(i,j) = dht_overlap_calc(old,j,new,i);
+                }
+        }
+
+	for (i = 0; i < new->cnt; i++) {
+		if (new->list[i].err > 0) {
+			/* Subvol might be marked for decommission
+			   with EINVAL, or some other serious error
+			   marked with positive errno.
+			*/
+			continue;
+                }
+
+                max_overlap = 0;
+                max_overlap_idx = i;
+                for (j = (i + 1); j < new->cnt; ++j) {
+                        if (new->list[j].err > 0) {
+			        /* Subvol might be marked for decommission
+			        with EINVAL, or some other serious error
+			        marked with positive errno.
+			        */
+			        continue;
+                        }
+                        /* Calculate the overlap now. */
+                        curr_overlap = OV_ENTRY(i,i) + OV_ENTRY(j,j);
+                        /* Calculate the overlap after the proposed swap. */
+                        overlap = OV_ENTRY(i,j) + OV_ENTRY(j,i);
+                        /* Are we better than status quo? */
+                        if (overlap > curr_overlap) {
+                                overlap -= curr_overlap;
+                                /* Are we better than the previous choice? */
+                                if (overlap > max_overlap) {
+                                        max_overlap = overlap;
+                                        max_overlap_idx = j;
+                                }
+                        }
+                }
+
+		if (max_overlap_idx != i) {
+ 			dht_layout_range_swap (new, i, max_overlap_idx);
+                        /* Need to swap the table values too. */
+                        for (j = 0; j < old->cnt; ++j) {
+                                overlap = OV_ENTRY(i,j);
+                                OV_ENTRY(i,j) = OV_ENTRY(max_overlap_idx,j);
+                                OV_ENTRY(max_overlap_idx,j) = overlap;
+                        }
+                }
 	}
+}
 
-	chunk = ((unsigned long) 0xffffffff) / cnt;
-
-	start = 0;
-	for (i = 0; i < layout->cnt; i++) {
-		err = layout->list[i].err;
-		if (err == -1) {
-			layout->list[i].start = start;
-			layout->list[i].stop  = start + chunk - 1;
-			
-			start = start + chunk;
-
-			gf_log (this->name, GF_LOG_DEBUG,
-				"gave fix: %u - %u on %s for %s",
-				layout->list[i].start, layout->list[i].stop,
-				layout->list[i].xlator->name, loc->path);
-			if (--cnt == 0) {
-				layout->list[i].stop = 0xffffffff;
-				break;
-			}
-		}
-	}
+
+dht_layout_t *
+dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
+                             dht_layout_t *layout)
+{
+        int           i            = 0;
+        xlator_t     *this         = NULL;
+        dht_layout_t *new_layout   = NULL;
+        dht_conf_t   *priv         = NULL;
+        dht_local_t  *local        = NULL;
+        uint32_t      subvol_down  = 0;
+        int           ret          = 0;
+
+        this  = frame->this;
+        priv  = this->private;
+        local = frame->local;
+
+        if (layout->type == DHT_HASH_TYPE_DM_USER) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "leaving %s alone",
+                        loc->path);
+                goto done;
+        }
+
+        new_layout = dht_layout_new (this, priv->subvolume_cnt);
+        if (!new_layout)
+                goto done;
+
+        /* If a subvolume is down, do not re-write the layout. */
+        ret = dht_layout_anomalies (this, loc, layout, NULL, NULL, NULL,
+                                    &subvol_down, NULL, NULL);
+
+        if (subvol_down || (ret == -1)) {
+                gf_log (this->name, GF_LOG_WARNING, "%u subvolume(s) are down"
+                        ". Skipping fix layout.", subvol_down);
+                GF_FREE (new_layout);
+                return NULL;
+        }
+
+        for (i = 0; i < new_layout->cnt; i++) {
+		if (layout->list[i].err != ENOSPC)
+			new_layout->list[i].err = layout->list[i].err;
+		else
+			new_layout->list[i].err = -1;
+
+		new_layout->list[i].xlator = layout->list[i].xlator;
+        }
+
+	/* First give it a layout as though it is a new directory. This
+	   ensures rotation to kick in */
+        dht_layout_sort_volname (new_layout);
+	dht_selfheal_layout_new_directory (frame, loc, new_layout);
+
+	/* Now selectively re-assign ranges only when it helps */
+	dht_selfheal_layout_maximize_overlap (frame, loc, new_layout, layout);
+
+done:
+        if (new_layout) {
+                /* Now that the new layout has all the proper layout, change the
+                   inode context */
+                dht_layout_set (this, loc->inode, new_layout);
+
+                /* Make sure the extra 'ref' for existing layout is removed */
+                dht_layout_unref (this, local->layout);
+
+                local->layout = new_layout;
+        }
+
+        return local->layout;
 }
 
 
+void
+dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
+                                   dht_layout_t *layout)
+{
+        xlator_t    *this = NULL;
+        uint32_t     chunk = 0;
+        int          i = 0;
+        uint32_t     start = 0;
+        int          cnt = 0;
+        int          err = 0;
+        int          start_subvol = 0;
+
+        this = frame->this;
+
+        cnt = dht_get_layout_count (this, layout, 1);
+
+        chunk = ((unsigned long) 0xffffffff) / ((cnt) ? cnt : 1);
+
+        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
+
+        /* clear out the range, as we are re-computing here */
+        DHT_RESET_LAYOUT_RANGE (layout);
+        for (i = start_subvol; i < layout->cnt; i++) {
+                err = layout->list[i].err;
+                if (err == -1 || err == ENOENT) {
+                        DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
+                                             cnt, loc->path);
+                        if (--cnt == 0) {
+                                layout->list[i].stop = 0xffffffff;
+                                goto done;
+                        }
+                        start += chunk;
+                }
+        }
+
+        for (i = 0; i < start_subvol; i++) {
+                err = layout->list[i].err;
+                if (err == -1 || err == ENOENT) {
+                        DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
+                                             cnt, loc->path);
+                        if (--cnt == 0) {
+                                layout->list[i].stop = 0xffffffff;
+                                goto done;
+                        }
+                        start += chunk;
+                }
+        }
+
+done:
+        return;
+}
+
 int
 dht_selfheal_dir_getafix (call_frame_t *frame, loc_t *loc,
-			  dht_layout_t *layout)
+                          dht_layout_t *layout)
 {
-	dht_conf_t  *conf = NULL;
-	xlator_t    *this = NULL;
-	dht_local_t *local = NULL;
-	int          missing = -1;
-	int          down = -1;
-	int          holes = -1;
-	int          ret = -1;
-	int          i = -1;
-	int          overlaps = -1;
-
-	this = frame->this;
-	conf = this->private;
-	local = frame->local;
-
-	missing = local->selfheal.missing;
-	down = local->selfheal.down;
-	holes = local->selfheal.hole_cnt;
-	overlaps = local->selfheal.overlaps_cnt;
-
-	if ((missing + down) == conf->subvolume_cnt) {
-		dht_selfheal_layout_new_directory (frame, loc, layout);
-		ret = 0;
-	}
+        dht_local_t *local = NULL;
+        uint32_t     holes = 0;
+        int          ret = -1;
+        int          i = -1;
+        uint32_t     overlaps = 0;
 
-	if (holes <= down) {
-		/* the down subvol might fill up the holes */
-		ret = 0;
-	}
+        local = frame->local;
 
-	if (holes || missing || overlaps) {
-		dht_selfheal_layout_new_directory (frame, loc, layout);
-		ret = 0;
-	}
+        holes = local->selfheal.hole_cnt;
+        overlaps = local->selfheal.overlaps_cnt;
 
-	for (i = 0; i < layout->cnt; i++) {
-		/* directory not present */
-		if (layout->list[i].err == ENOENT) {
-			ret = 0;
-			break;
-		}
-	}
+        if (holes || overlaps) {
+                dht_selfheal_layout_new_directory (frame, loc, layout);
+                ret = 0;
+        }
 
-	/* TODO: give a fix to these non-virgins */
+        for (i = 0; i < layout->cnt; i++) {
+                /* directory not present */
+                if (layout->list[i].err == ENOENT) {
+                        ret = 0;
+                        break;
+                }
+        }
 
-	return ret;
+        /* TODO: give a fix to these non-virgins */
+
+        return ret;
 }
 
 int
-dht_selfheal_new_directory (call_frame_t *frame, 
-			    dht_selfheal_dir_cbk_t dir_cbk,
-			    dht_layout_t *layout)
+dht_selfheal_new_directory (call_frame_t *frame,
+                            dht_selfheal_dir_cbk_t dir_cbk,
+                            dht_layout_t *layout)
 {
-	dht_local_t *local = NULL;
+        dht_local_t *local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	local->selfheal.dir_cbk = dir_cbk;
-	local->selfheal.layout = layout;
+        local->selfheal.dir_cbk = dir_cbk;
+        local->selfheal.layout = dht_layout_ref (frame->this, layout);
 
-	dht_layout_sort_volname (layout);
-	dht_selfheal_layout_new_directory (frame, &local->loc, layout);	
-	dht_selfheal_dir_xattr (frame, &local->loc, layout);
-	return 0;
+        dht_layout_sort_volname (layout);
+        dht_selfheal_layout_new_directory (frame, &local->loc, layout);
+        dht_selfheal_dir_xattr (frame, &local->loc, layout);
+        return 0;
 }
 
-
 int
-dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-			loc_t *loc, dht_layout_t *layout)
+dht_fix_directory_layout (call_frame_t *frame,
+                          dht_selfheal_dir_cbk_t dir_cbk,
+                          dht_layout_t *layout)
 {
-	dht_local_t *local    = NULL;
-	uint32_t     holes    = 0;
-	uint32_t     overlaps = 0;
-	uint32_t     missing  = 0;
-	uint32_t     down     = 0;
-	uint32_t     misc     = 0;
-	int          ret      = 0;
-	xlator_t    *this     = NULL;
-
-	local = frame->local;
-	this = frame->this;
-
-	ret = dht_layout_anomalies (this, loc, layout,
-				    &local->selfheal.hole_cnt,
-				    &local->selfheal.overlaps_cnt,
-				    &local->selfheal.missing,
-				    &local->selfheal.down,
-				    &local->selfheal.misc);
-
-	holes    = local->selfheal.hole_cnt;
-	overlaps = local->selfheal.overlaps_cnt;
-	missing  = local->selfheal.missing;
-	down     = local->selfheal.down;
-	misc     = local->selfheal.misc;
-
-	local->selfheal.dir_cbk = dir_cbk;
-	local->selfheal.layout = layout;
+        dht_local_t  *local = NULL;
+        dht_layout_t *tmp_layout = NULL;
 
-/*
-	if (down) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%d subvolumes down -- not fixing", down);
-		ret = 0;
-		goto sorry_no_fix;
-	}
-
-	if (overlaps) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"not fixing overlaps in %s", loc->path);
-		local->op_errno = EINVAL;
-		ret = -1;
-		goto sorry_no_fix;
-	}
+        local = frame->local;
 
-	if (misc) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%d subvolumes have unrecoverable errors", misc);
-		ret = 0;
-		goto sorry_no_fix;
-	}
+        local->selfheal.dir_cbk = dir_cbk;
+        local->selfheal.layout = dht_layout_ref (frame->this, layout);
 
-	if (holes > missing) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%d holes and %d pigeons -- not fixing",
-			holes, missing);
-		ret = 0;
-		goto sorry_no_fix;
-	}
-*/
-	dht_layout_sort_volname (layout);
-	ret = dht_selfheal_dir_getafix (frame, loc, layout);
+        /* No layout sorting required here */
+        tmp_layout = dht_fix_layout_of_directory (frame, &local->loc, layout);
+        if (!tmp_layout) {
+                return -1;
+        }
+        dht_fix_dir_xattr (frame, &local->loc, tmp_layout);
 
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"not able to form layout for the directory");
-		goto sorry_no_fix;
-	}
+        return 0;
+}
 
-	dht_selfheal_dir_mkdir (frame, loc, layout, 0);
 
-	return 0;
+int
+dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                        loc_t *loc, dht_layout_t *layout)
+{
+        dht_local_t *local    = NULL;
+        uint32_t     down     = 0;
+        uint32_t     misc     = 0;
+        int          ret      = 0;
+        xlator_t    *this     = NULL;
+
+        local = frame->local;
+        this = frame->this;
+
+        dht_layout_anomalies (this, loc, layout,
+                              &local->selfheal.hole_cnt,
+                              &local->selfheal.overlaps_cnt,
+                              NULL, &local->selfheal.down,
+                              &local->selfheal.misc, NULL);
+
+        down     = local->selfheal.down;
+        misc     = local->selfheal.misc;
+
+        local->selfheal.dir_cbk = dir_cbk;
+        local->selfheal.layout = dht_layout_ref (this, layout);
+
+        if (down) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%d subvolumes down -- not fixing", down);
+                ret = 0;
+                goto sorry_no_fix;
+        }
+
+        if (misc) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%d subvolumes have unrecoverable errors", misc);
+                ret = 0;
+                goto sorry_no_fix;
+        }
+
+        dht_layout_sort_volname (layout);
+        ret = dht_selfheal_dir_getafix (frame, loc, layout);
+
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "not able to form layout for the directory");
+                goto sorry_no_fix;
+        }
+
+        dht_selfheal_dir_mkdir (frame, loc, layout, 0);
+
+        return 0;
 
 sorry_no_fix:
-	/* TODO: need to put appropriate local->op_errno */
-	dht_selfheal_dir_finish (frame, this, ret);
+        /* TODO: need to put appropriate local->op_errno */
+        dht_selfheal_dir_finish (frame, this, ret);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-		      loc_t *loc, dht_layout_t *layout)
+                      loc_t *loc, dht_layout_t *layout)
 {
-	int          ret = 0;
-	dht_local_t *local    = NULL;
+        int          ret = 0;
+        dht_local_t *local    = NULL;
 
+        local = frame->local;
 
-	local = frame->local;
+        local->selfheal.dir_cbk = dir_cbk;
+        local->selfheal.layout = dht_layout_ref (frame->this, layout);
 
-	local->selfheal.dir_cbk = dir_cbk;
-	local->selfheal.layout = layout;
+        ret = dht_selfheal_dir_mkdir (frame, loc, layout, 1);
 
-	ret = dht_selfheal_dir_mkdir (frame, loc, layout, 1);
+        return ret;
+}
 
-	return 0;
+int
+dht_dir_attr_heal (void *data)
+{
+        call_frame_t    *frame = NULL;
+        dht_local_t     *local = NULL;
+        xlator_t        *subvol = NULL;
+        xlator_t        *this  = NULL;
+        dht_conf_t      *conf  = NULL;
+        int              call_cnt = 0;
+        int              ret   = -1;
+        int              i     = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", data, out);
+
+        frame = data;
+        local = frame->local;
+        this = frame->this;
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", local, out);
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO ("dht", conf, out);
+
+        call_cnt = conf->subvolume_cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                subvol = conf->subvolumes[i];
+                if (!subvol || (subvol == dht_first_up_subvol (this)))
+                        continue;
+                ret = syncop_setattr (subvol, &local->loc, &local->stbuf,
+                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                                      NULL, NULL);
+                if (ret)
+                        gf_log ("dht", GF_LOG_ERROR, "Failed to set uid/gid on"
+                                " %s on %s subvol (%s)", local->loc.path,
+                                subvol->name, strerror (errno));
+        }
+out:
+        return 0;
+}
+
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data)
+{
+        DHT_STACK_DESTROY (sync_frame);
+        return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
new file mode 100644
index 000000000..70aac7710
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -0,0 +1,758 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+/* TODO: add NS locking */
+
+#include "statedump.h"
+#include "dht-common.h"
+
+/* TODO:
+   - use volumename in xattr instead of "dht"
+   - use NS locks
+   - handle all cases in self heal layout reconstruction
+   - complete linkfile selfheal
+*/
+struct volume_options options[];
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix)
+{
+
+        char    key[GF_DUMP_MAX_BUF_LEN];
+        int     i = 0;
+
+        if (!layout)
+                goto out;
+        if (!prefix)
+                goto out;
+
+        gf_proc_dump_build_key(key, prefix, "cnt");
+        gf_proc_dump_write(key, "%d", layout->cnt);
+        gf_proc_dump_build_key(key, prefix, "preset");
+        gf_proc_dump_write(key, "%d", layout->preset);
+        gf_proc_dump_build_key(key, prefix, "gen");
+        gf_proc_dump_write(key, "%d", layout->gen);
+        if (layout->type != IA_INVAL) {
+                gf_proc_dump_build_key(key, prefix, "inode type");
+                gf_proc_dump_write(key, "%d", layout->type);
+        }
+
+        if  (!IA_ISDIR (layout->type))
+                goto out;
+
+        for (i = 0; i < layout->cnt; i++) {
+                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
+                gf_proc_dump_write(key, "%d", layout->list[i].err);
+                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].start);
+                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].stop);
+                if (layout->list[i].xlator) {
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.type", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->type);
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.name", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->name);
+                }
+        }
+
+out:
+        return;
+}
+
+
+int32_t
+dht_priv_dump (xlator_t *this)
+{
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        char            key[GF_DUMP_MAX_BUF_LEN];
+        int             i = 0;
+        dht_conf_t      *conf = NULL;
+        int             ret = -1;
+
+        if (!this)
+                goto out;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        ret = TRY_LOCK(&conf->subvolume_lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
+                               this->name);
+        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                sprintf (key, "subvolumes[%d]", i);
+                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
+                                   conf->subvolumes[i]->name);
+                if (conf->file_layouts && conf->file_layouts[i]){
+                        sprintf (key, "file_layouts[%d]", i);
+                        dht_layout_dump(conf->file_layouts[i], key);
+                }
+                if (conf->dir_layouts && conf->dir_layouts[i]) {
+                        sprintf (key, "dir_layouts[%d]", i);
+                        dht_layout_dump(conf->dir_layouts[i], key);
+                }
+                if (conf->subvolume_status) {
+
+                        sprintf (key, "subvolume_status[%d]", i);
+                        gf_proc_dump_write(key, "%d",
+                                           (int)conf->subvolume_status[i]);
+                }
+
+        }
+
+        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
+        gf_proc_dump_write("gen", "%d", conf->gen);
+        gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
+	gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
+        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
+        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
+        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
+        if (conf ->du_stats) {
+                gf_proc_dump_write("du_stats.avail_percent", "%lf",
+                                   conf->du_stats->avail_percent);
+                gf_proc_dump_write("du_stats.avail_space", "%lu",
+                                   conf->du_stats->avail_space);
+		gf_proc_dump_write("du_stats.avail_inodes", "%lf",
+                                   conf->du_stats->avail_inodes);
+                gf_proc_dump_write("du_stats.log", "%lu", conf->du_stats->log);
+        }
+
+        if (conf->last_stat_fetch.tv_sec)
+                gf_proc_dump_write("last_stat_fetch", "%s",
+                                    ctime(&conf->last_stat_fetch.tv_sec));
+
+        UNLOCK(&conf->subvolume_lock);
+
+out:
+        return ret;
+}
+
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode)
+{
+        int             ret = -1;
+        dht_layout_t    *layout = NULL;
+
+        if (!this)
+                goto out;
+        if (!inode)
+                goto out;
+
+        ret = dht_inode_ctx_layout_get (inode, this, &layout);
+
+        if ((ret != 0) || !layout)
+                return ret;
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
+        dht_layout_dump(layout, "layout");
+
+out:
+        return ret;
+}
+
+void
+dht_fini (xlator_t *this)
+{
+        int         i = 0;
+        dht_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        conf = this->private;
+        this->private = NULL;
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf);
+        }
+out:
+        return;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        "failed");
+                return ret;
+        }
+out:
+        return ret;
+}
+
+
+int
+dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
+                                 const char *bricks)
+{
+        int         i  = 0;
+        int         ret  = -1;
+        char       *tmpstr = NULL;
+        char       *dup_brick = NULL;
+        char       *node = NULL;
+
+        if (!conf || !bricks)
+                goto out;
+
+        dup_brick = gf_strdup (bricks);
+        node = strtok_r (dup_brick, ",", &tmpstr);
+        while (node) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (!strcmp (conf->subvolumes[i]->name, node)) {
+                                conf->decommissioned_bricks[i] =
+                                        conf->subvolumes[i];
+                                        conf->decommission_subvols_cnt++;
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "decommissioning subvolume %s",
+                                        conf->subvolumes[i]->name);
+                                break;
+                        }
+                }
+                if (i == conf->subvolume_cnt) {
+                        /* Wrong node given. */
+                        goto out;
+                }
+                node = strtok_r (NULL, ",", &tmpstr);
+        }
+
+        ret = 0;
+        conf->decommission_in_progress = 1;
+out:
+        GF_FREE (dup_brick);
+
+        return ret;
+}
+
+
+int
+dht_decommissioned_remove (xlator_t *this, dht_conf_t *conf)
+{
+        int         i  = 0;
+        int         ret  = -1;
+
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->decommissioned_bricks[i]) {
+                        conf->decommissioned_bricks[i] = NULL;
+                        conf->decommission_subvols_cnt--;
+                }
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+void
+dht_init_regex (xlator_t *this, dict_t *odict, char *name,
+                regex_t *re, gf_boolean_t *re_valid)
+{
+        char    *temp_str;
+
+        if (dict_get_str (odict, name, &temp_str) != 0) {
+                if (strcmp(name,"rsync-hash-regex")) {
+                        return;
+                }
+                temp_str = "^\\.(.+)\\.[^.]+$";
+        }
+
+        if (*re_valid) {
+                regfree(re);
+                *re_valid = _gf_false;
+        }
+
+        if (!strcmp(temp_str,"none")) {
+                return;
+        }
+
+        if (regcomp(re,temp_str,REG_EXTENDED) == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "using regex %s = %s", name, temp_str);
+                *re_valid = _gf_true;
+        }
+        else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "compiling regex %s failed", temp_str);
+        }
+}
+
+int
+dht_reconfigure (xlator_t *this, dict_t *options)
+{
+        dht_conf_t      *conf = NULL;
+        char            *temp_str = NULL;
+        gf_boolean_t     search_unhashed;
+        int              ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", options, out);
+
+        conf = this->private;
+        if (!conf)
+                return 0;
+
+        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean*/
+                if (strcasecmp (temp_str, "auto")) {
+                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
+                                gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                                       " lookup-unhashed reconfigured (%s)",
+                                       temp_str);
+                                conf->search_unhashed = search_unhashed;
+                        } else {
+                                gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
+                                       " lookup-unhashed should be boolean,"
+                                       " not (%s), defaulting to (%d)",
+                                       temp_str, conf->search_unhashed);
+                                //return -1;
+                                ret = -1;
+                                goto out;
+                        }
+                } else {
+                        gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                               " lookup-unhashed reconfigured auto ");
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+                }
+        }
+
+	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
+                          percent_or_size, out);
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100.0)
+                conf->disk_unit = 'p';
+
+	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
+                          percent, out);
+
+        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
+                          options, uint32, out);
+
+        GF_OPTION_RECONF ("readdir-optimize", conf->readdir_optimize, options,
+                          bool, out);
+        if (conf->defrag) {
+                GF_OPTION_RECONF ("rebalance-stats", conf->defrag->stats,
+                                  options, bool, out);
+        }
+
+        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto out;
+        } else {
+                ret = dht_decommissioned_remove (this, conf);
+                if (ret == -1)
+                        goto out;
+        }
+
+        dht_init_regex (this, options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+gf_defrag_pattern_list_fill (xlator_t *this, gf_defrag_info_t *defrag, char *data)
+{
+        int                    ret = -1;
+        char                  *tmp_str = NULL;
+        char                  *tmp_str1 = NULL;
+        char                  *dup_str = NULL;
+        char                  *num = NULL;
+        char                  *pattern_str = NULL;
+        char                  *pattern = NULL;
+        gf_defrag_pattern_list_t *temp_list = NULL;
+        gf_defrag_pattern_list_t *pattern_list = NULL;
+
+        if (!this || !defrag || !data)
+                goto out;
+
+        /* Get the pattern for pattern list. "pattern:<optional-size>"
+         * eg: *avi, *pdf:10MB, *:1TB
+         */
+        pattern_str = strtok_r (data, ",", &tmp_str);
+        while (pattern_str) {
+                dup_str = gf_strdup (pattern_str);
+                pattern_list = GF_CALLOC (1, sizeof (gf_defrag_pattern_list_t),
+                                        1);
+                if (!pattern_list) {
+                        goto out;
+                }
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                num = strtok_r (NULL, ":", &tmp_str1);
+                if (!pattern)
+                        goto out;
+                if (!num) {
+                        if (gf_string2bytesize(pattern, &pattern_list->size)
+                             == 0) {
+                                pattern = "*";
+                        }
+                } else if (gf_string2bytesize (num, &pattern_list->size) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format \"%s\"", num);
+                        goto out;
+                }
+                memcpy (pattern_list->path_pattern, pattern, strlen (dup_str));
+
+                if (!defrag->defrag_pattern)
+                        temp_list = NULL;
+                else
+                        temp_list = defrag->defrag_pattern;
+
+                pattern_list->next = temp_list;
+
+                defrag->defrag_pattern = pattern_list;
+                pattern_list = NULL;
+
+                GF_FREE (dup_str);
+                dup_str = NULL;
+
+                pattern_str = strtok_r (NULL, ",", &tmp_str);
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                GF_FREE (pattern_list);
+        GF_FREE (dup_str);
+
+        return ret;
+}
+
+int
+dht_init (xlator_t *this)
+{
+        dht_conf_t                      *conf           = NULL;
+        char                            *temp_str       = NULL;
+        int                              ret            = -1;
+        int                              i              = 0;
+        gf_defrag_info_t                *defrag         = NULL;
+        int                              cmd            = 0;
+        char                            *node_uuid      = NULL;
+
+
+        GF_VALIDATE_OR_GOTO ("dht", this, err);
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Distribute needs more than one subvolume");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile");
+        }
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
+        if (!conf) {
+                goto err;
+        }
+
+        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
+
+        if (cmd) {
+                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
+                                    gf_defrag_info_mt);
+
+                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
+
+                LOCK_INIT (&defrag->lock);
+
+                defrag->is_exiting = 0;
+
+                conf->defrag = defrag;
+
+                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "node-uuid not "
+                                "specified");
+                        goto err;
+                }
+
+                if (uuid_parse (node_uuid, defrag->node_uuid)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot parse "
+                                "glusterd node uuid");
+                        goto err;
+                }
+
+                defrag->cmd = cmd;
+
+                defrag->stats = _gf_false;
+        }
+
+        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
+        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean */
+                if (strcasecmp (temp_str, "auto"))
+                        gf_string2boolean (temp_str, &conf->search_unhashed);
+                else
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+
+        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
+                        err);
+
+        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
+
+	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
+			err);
+
+        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
+                        err);
+
+        conf->dir_spread_cnt = conf->subvolume_cnt;
+        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
+                        uint32, err);
+
+        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
+                        bool, err);
+
+        GF_OPTION_INIT ("readdir-optimize", conf->readdir_optimize, bool, err);
+
+        if (defrag) {
+                GF_OPTION_INIT ("rebalance-stats", defrag->stats, bool, err);
+                if (dict_get_str (this->options, "rebalance-filter", &temp_str)
+                    == 0) {
+                        if (gf_defrag_pattern_list_fill (this, defrag, temp_str)
+                            == -1) {
+                                gf_log (this->name, GF_LOG_ERROR, "Cannot parse"
+                                        " rebalance-filter (%s)", temp_str);
+                                goto err;
+                        }
+                }
+        }
+
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100)
+                conf->disk_unit = 'p';
+
+        ret = dht_init_subvolumes (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto err;
+        }
+
+        dht_init_regex (this, this->options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, this->options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = dht_layouts_init (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        LOCK_INIT (&conf->subvolume_lock);
+        LOCK_INIT (&conf->layout_lock);
+
+        conf->gen = 1;
+
+        this->local_pool = mem_pool_new (dht_local_t, 512);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto err;
+        }
+
+        GF_OPTION_INIT ("xattr-name", conf->xattr_name, str, err);
+        gf_asprintf (&conf->link_xattr_name, "%s.linkto", conf->xattr_name);
+        gf_asprintf (&conf->wild_xattr_name, "%s*", conf->xattr_name);
+        if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+                goto err;
+        }
+
+        this->private = conf;
+
+        return 0;
+
+err:
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf->du_stats);
+
+                GF_FREE (conf->defrag);
+
+                GF_FREE (conf->xattr_name);
+                GF_FREE (conf->link_xattr_name);
+                GF_FREE (conf->wild_xattr_name);
+
+                GF_FREE (conf);
+        }
+
+        return -1;
+}
+
+
+struct volume_options options[] = {
+        { .key  = {"lookup-unhashed"},
+          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
+                    "on", "off"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "on",
+          .description = "This option if set to ON, does a lookup through "
+          "all the sub-volumes, in case a lookup didn't return any result "
+          "from the hash subvolume. If set to OFF, it does not do a lookup "
+          "on the remaining subvolumes."
+        },
+        { .key  = {"min-free-disk"},
+          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
+          .default_value = "10%",
+          .description = "Percentage/Size of disk space, after which the "
+          "process starts balancing out the cluster, and logs will appear "
+          "in log files",
+        },
+	{ .key  = {"min-free-inodes"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .default_value = "5%",
+          .description = "after system has only N% of inodes, warnings "
+          "starts to appear in log files",
+        },
+        { .key = {"unhashed-sticky-bit"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+        { .key = {"use-readdirp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option if set to ON, forces the use of "
+          "readdirp, and hence also displays the stats of the files."
+        },
+        { .key = {"assert-no-child-down"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON, in the event of "
+          "CHILD_DOWN, will call exit."
+        },
+        { .key  = {"directory-layout-spread"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "Specifies the directory layout spread."
+        },
+        { .key  = {"decommissioned-bricks"},
+          .type = GF_OPTION_TYPE_ANY,
+          .description = "This option if set to ON, decommissions "
+          "the brick, so that no new data is allowed to be created "
+          "on that brick."
+        },
+        { .key  = {"rebalance-cmd"},
+          .type = GF_OPTION_TYPE_INT,
+        },
+        { .key = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+        { .key = {"rebalance-stats"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON displays and logs the "
+          " time taken for migration of each file, during the rebalance "
+          "process. If set to OFF, the rebalance logs will only display the "
+          "time spent in each directory."
+        },
+        { .key = {"readdir-optimize"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON enables the optimization "
+          "that allows DHT to requests non-first subvolumes to filter out "
+          "directory entries."
+        },
+        { .key = {"rsync-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by rsync, to prevent relocation when the "
+          "file is renamed."
+        },
+        { .key = {"extra-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by an application, to prevent relocation when "
+          "the file is renamed."
+        },
+        { .key = {"rebalance-filter"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
+        { .key = {"xattr-name"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "trusted.glusterfs.dht",
+          .description = "Base for extended attributes used by this "
+          "translator instance, to avoid conflicts with others above or "
+          "below it."
+        },
+
+        /* NUFA option */
+        { .key  = {"local-volume-name"},
+          .type = GF_OPTION_TYPE_XLATOR
+        },
+
+        /* switch option */
+        { .key  = {"pattern.switch.case"},
+          .type = GF_OPTION_TYPE_ANY
+        },
+
+        { .key  = {NULL} },
+};
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 836e7a4e8..fc0ca2f77 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -23,200 +14,76 @@
 #include "config.h"
 #endif
 
-/* TODO: add NS locking */
-
-#include "dht-common.c"
-
-/* TODO:
-   - use volumename in xattr instead of "dht"
-   - use NS locks
-   - handle all cases in self heal layout reconstruction
-   - complete linkfile selfheal
-*/
-
-
-
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-	int ret = -1;
-
-	ret = dht_notify (this, event, data);
-
-	return ret;
-}
-
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
-
-	conf = this->private;
-
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                FREE (conf->file_layouts[i]);
-                        }
-                        FREE (conf->file_layouts);
-                }
-
-                if (conf->default_dir_layout)
-                        FREE (conf->default_dir_layout);
-
-                if (conf->subvolumes)
-                        FREE (conf->subvolumes);
-
-		if (conf->subvolume_status)
-			FREE (conf->subvolume_status);
-
-                FREE (conf);
-        }
-
-	return;
-}
-
-int
-init (xlator_t *this)
-{
-        dht_conf_t    *conf = NULL;
-	char          *lookup_unhashed_str = NULL;
-        int            ret = -1;
-        int            i = 0;
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"DHT needs more than one child defined");
-		return -1;
-	}
-  
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-        conf = CALLOC (1, sizeof (*conf));
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "memory allocation failed :(");
-                goto err;
-        }
-
-	conf->search_unhashed = 0;
-
-	if (dict_get_str (this->options, "lookup-unhashed",
-			  &lookup_unhashed_str) == 0) {
-		gf_string2boolean (lookup_unhashed_str,
-				   &conf->search_unhashed);
-	}
-
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-	LOCK_INIT (&conf->subvolume_lock);
-
-	conf->gen = 1;
-
-        this->private = conf;
-
-        return 0;
-
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                FREE (conf->file_layouts[i]);
-                        }
-                        FREE (conf->file_layouts);
-                }
-
-                if (conf->default_dir_layout)
-                        FREE (conf->default_dir_layout);
-
-                if (conf->subvolumes)
-                        FREE (conf->subvolumes);
-
-		if (conf->subvolume_status)
-			FREE (conf->subvolume_status);
-
-                FREE (conf);
-        }
-
-        return -1;
-}
+#include "statedump.h"
+#include "dht-common.h"
 
+class_methods_t class_methods = {
+        .init           = dht_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
 
 struct xlator_fops fops = {
-	.lookup      = dht_lookup,
-	.mknod       = dht_mknod,
-	.create      = dht_create,
-
-	.stat        = dht_stat,
-	.chmod       = dht_chmod,
-	.chown       = dht_chown,
-	.fchown      = dht_fchown,
-	.fchmod      = dht_fchmod,
-	.fstat       = dht_fstat,
-	.utimens     = dht_utimens,
-	.truncate    = dht_truncate,
-	.ftruncate   = dht_ftruncate,
-	.access      = dht_access,
-	.readlink    = dht_readlink,
-	.setxattr    = dht_setxattr,
-	.getxattr    = dht_getxattr,
-	.removexattr = dht_removexattr,
-	.open        = dht_open,
-	.readv       = dht_readv,
-	.writev      = dht_writev,
-	.flush       = dht_flush,
-	.fsync       = dht_fsync,
-	.statfs      = dht_statfs,
-	.lk          = dht_lk,
-	.opendir     = dht_opendir,
-	.readdir     = dht_readdir,
-	.fsyncdir    = dht_fsyncdir,
-	.symlink     = dht_symlink,
-	.unlink      = dht_unlink,
-	.link        = dht_link,
-	.mkdir       = dht_mkdir,
-	.rmdir       = dht_rmdir,
-	.rename      = dht_rename,
-	.inodelk     = dht_inodelk,
-	.finodelk    = dht_finodelk,
-	.entrylk     = dht_entrylk,
-	.fentrylk    = dht_fentrylk,
-	.xattrop     = dht_xattrop,
-	.fxattrop    = dht_fxattrop,
-#if 0
-	.setdents    = dht_setdents,
-	.getdents    = dht_getdents,
-	.checksum    = dht_checksum,
-#endif
+        .lookup      = dht_lookup,
+        .mknod       = dht_mknod,
+        .create      = dht_create,
+
+        .open        = dht_open,
+        .statfs      = dht_statfs,
+        .opendir     = dht_opendir,
+        .readdir     = dht_readdir,
+        .readdirp    = dht_readdirp,
+        .fsyncdir    = dht_fsyncdir,
+        .symlink     = dht_symlink,
+        .unlink      = dht_unlink,
+        .link        = dht_link,
+        .mkdir       = dht_mkdir,
+        .rmdir       = dht_rmdir,
+        .rename      = dht_rename,
+        .entrylk     = dht_entrylk,
+        .fentrylk    = dht_fentrylk,
+
+        /* Inode read operations */
+        .stat        = dht_stat,
+        .fstat       = dht_fstat,
+        .access      = dht_access,
+        .readlink    = dht_readlink,
+        .getxattr    = dht_getxattr,
+        .fgetxattr    = dht_fgetxattr,
+        .readv       = dht_readv,
+        .flush       = dht_flush,
+        .fsync       = dht_fsync,
+        .inodelk     = dht_inodelk,
+        .finodelk    = dht_finodelk,
+        .lk          = dht_lk,
+
+        /* Inode write operations */
+        .fremovexattr = dht_fremovexattr,
+        .removexattr = dht_removexattr,
+        .setxattr    = dht_setxattr,
+        .fsetxattr   = dht_fsetxattr,
+        .truncate    = dht_truncate,
+        .ftruncate   = dht_ftruncate,
+        .writev      = dht_writev,
+        .xattrop     = dht_xattrop,
+        .fxattrop    = dht_fxattrop,
+        .setattr     = dht_setattr,
+        .fsetattr    = dht_fsetattr,
+	.fallocate   = dht_fallocate,
+	.discard     = dht_discard,
+        .zerofill    = dht_zerofill,
 };
 
-
-struct xlator_mops mops = {
+struct xlator_dumpops dumpops = {
+        .priv = dht_priv_dump,
+        .inodectx = dht_inodectx_dump,
 };
 
 
 struct xlator_cbks cbks = {
-//	.release    = dht_release,
+//      .release    = dht_release,
 //      .releasedir = dht_releasedir,
-	.forget     = dht_forget
-};
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"}, 
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = {NULL} },
+        .forget     = dht_forget
 };
+;
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index 6333e002f..e934acdf0 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -23,16 +14,18 @@
 #include "config.h"
 #endif
 
-#include "dht-common.c"
+#include "dht-common.h"
 
 /* TODO: all 'TODO's in dht.c holds good */
 
-int 
+extern struct volume_options options[];
+
+int
 nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno,
-		       inode_t *inode, struct stat *stbuf, dict_t *xattr)
+                       int op_ret, int op_errno,
+                       inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                       struct iatt *postparent)
 {
-	dht_layout_t *layout      = NULL;
         xlator_t     *subvol      = NULL;
         char          is_linkfile = 0;
         char          is_dir      = 0;
@@ -41,8 +34,8 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         loc_t        *loc         = NULL;
         int           i           = 0;
         call_frame_t *prev        = NULL;
-	int           call_cnt    = 0;
-
+        int           call_cnt    = 0;
+        int           ret         = 0;
 
         conf  = this->private;
 
@@ -50,58 +43,52 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local = frame->local;
         loc   = &local->loc;
 
-	if (ENTRY_MISSING (op_ret, op_errno)) {
-		if (conf->search_unhashed) {
-			local->op_errno = ENOENT;
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
-		}
-	}
+        if (ENTRY_MISSING (op_ret, op_errno)) {
+                if (conf->search_unhashed) {
+                        local->op_errno = ENOENT;
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
+                }
+        }
 
         if (op_ret == -1)
                 goto out;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
         is_dir      = check_is_dir (inode, stbuf, xattr);
 
         if (!is_dir && !is_linkfile) {
                 /* non-directory and not a linkfile */
+                ret = dht_layout_preset (this, prev->this, inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "could not set pre-set layout for subvol %s",
+                                prev->this->name);
+                        op_ret   = -1;
+                        op_errno = EINVAL;
+                        goto err;
+                }
 
-		dht_itransform (this, prev->this, stbuf->st_ino,
-				&stbuf->st_ino);
-
-		layout = dht_layout_for_subvol (this, prev->this);
-		if (!layout) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"no pre-set layout for subvolume %s",
-				prev->this->name);
-			op_ret   = -1;
-			op_errno = EINVAL;
-			goto err;
-		}
-
-                inode_ctx_put (inode, this, (uint64_t)(long)layout);
                 goto out;
         }
 
         if (is_dir) {
                 call_cnt        = conf->subvolume_cnt;
-		local->call_cnt = call_cnt;
+                local->call_cnt = call_cnt;
 
                 local->inode = inode_ref (inode);
                 local->xattr = dict_ref (xattr);
 
-		local->op_ret = 0;
-		local->op_errno = 0;
+                local->op_ret = 0;
+                local->op_errno = 0;
 
-		local->layout = dht_layout_new (this, conf->subvolume_cnt);
-		if (!local->layout) {
-			op_ret   = -1;
-			op_errno = ENOMEM;
-			gf_log (this->name, GF_LOG_ERROR,
-				"memory allocation failed :(");
-			goto err;
-		}
+                local->layout = dht_layout_new (this, conf->subvolume_cnt);
+                if (!local->layout) {
+                        op_ret   = -1;
+                        op_errno = ENOMEM;
+                        goto err;
+                }
 
                 for (i = 0; i < call_cnt; i++) {
                         STACK_WIND (frame, dht_lookup_dir_cbk,
@@ -115,54 +102,55 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
 
                 if (!subvol) {
-                        gf_log (this->name, GF_LOG_WARNING,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "linkfile not having link subvolume. path=%s",
                                 loc->path);
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
                 }
 
-		STACK_WIND (frame, dht_lookup_linkfile_cbk,
-			    subvol, subvol->fops->lookup,
-			    &local->loc, local->xattr_req);
+                STACK_WIND (frame, dht_lookup_linkfile_cbk,
+                            subvol, subvol->fops->lookup,
+                            &local->loc, local->xattr_req);
         }
 
         return 0;
 
 out:
-	if (!local->hashed_subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			local->loc.path);
-		op_errno = EINVAL;
-		goto err;
-	}
-		
-	STACK_WIND (frame, dht_lookup_cbk,
-		    local->hashed_subvol, local->hashed_subvol->fops->lookup,
-		    &local->loc, local->xattr_req);
-
-	return 0;
-
- err:
-        DHT_STACK_UNWIND (frame, op_ret, op_errno, inode, stbuf, xattr);
+        if (!local->hashed_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        local->loc.path);
+                local->op_errno = ENOENT;
+                dht_lookup_everywhere (frame, this, loc);
+                return 0;
+        }
+
+        STACK_WIND (frame, dht_lookup_cbk,
+                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                    &local->loc, local->xattr_req);
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
+                          inode, stbuf, xattr, postparent);
         return 0;
 }
 
 int
 nufa_lookup (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, dict_t *xattr_req)
+             loc_t *loc, dict_t *xattr_req)
 {
         xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
         xlator_t     *subvol = NULL;
         dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_conf_t   *conf = NULL;
         int           ret    = -1;
         int           op_errno = -1;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-	int           call_cnt = 0;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
 
 
         VALIDATE_OR_GOTO (frame, err);
@@ -171,514 +159,518 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc->inode, err);
         VALIDATE_OR_GOTO (loc->path, err);
 
-	conf = this->private;
-
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-        ret = loc_dup (loc, &local->loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "copying location failed for path=%s",
-                        loc->path);
+        conf = this->private;
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
+        if (!local) {
+                op_errno = ENOMEM;
                 goto err;
         }
 
-	if (xattr_req) {
-		local->xattr_req = dict_ref (xattr_req);
-	} else {
-		local->xattr_req = dict_new ();
-	}
+        if (xattr_req) {
+                local->xattr_req = dict_ref (xattr_req);
+        } else {
+                local->xattr_req = dict_new ();
+        }
 
-	hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-	cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
-	
-	local->cached_subvol = cached_subvol;
-	local->hashed_subvol = hashed_subvol;
+        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
 
-        if (is_revalidate (loc)) {
-		layout = dht_layout_get (this, loc->inode);
+        local->hashed_subvol = hashed_subvol;
 
+        if (is_revalidate (loc)) {
+                layout = local->layout;
                 if (!layout) {
-                        gf_log (this->name, GF_LOG_ERROR,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "revalidate without cache. path=%s",
                                 loc->path);
                         op_errno = EINVAL;
                         goto err;
                 }
 
-		if (layout->gen && (layout->gen < conf->gen)) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"incomplete layout failure for path=%s",
-				loc->path);
-			op_errno = EAGAIN;
-			goto err;
-		}
-
-		local->inode    = inode_ref (loc->inode);
-		local->st_ino   = loc->inode->ino;
-
-		local->call_cnt = layout->cnt;
-		call_cnt = local->call_cnt;
-		
-		/* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
-		 *       revalidates directly go to the cached-subvolume.
-		 */
-		ret = dict_set_uint32 (local->xattr_req, 
-				       "trusted.glusterfs.dht", 4 * 4);
-
-		for (i = 0; i < layout->cnt; i++) {
-			subvol = layout->list[i].xlator;
-			
-			STACK_WIND (frame, dht_revalidate_cbk,
-				    subvol, subvol->fops->lookup,
-				    loc, local->xattr_req);
-
-			if (!--call_cnt)
-				break;
-		}
-	} else {
-		ret = dict_set_uint32 (local->xattr_req, 
-				       "trusted.glusterfs.dht", 4 * 4);
-
-		ret = dict_set_uint32 (local->xattr_req, 
-				       "trusted.glusterfs.dht.linkto", 256);
-
-		/* Send it to only local volume */
-		STACK_WIND (frame, nufa_local_lookup_cbk,
-			    conf->local_volume, 
-			    conf->local_volume->fops->lookup,
-			    loc, local->xattr_req);
-	}
+                if (layout->gen && (layout->gen < conf->gen)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "incomplete layout failure for path=%s",
+                                loc->path);
+                        dht_layout_unref (this, local->layout);
+                        goto do_fresh_lookup;
+                }
+
+                local->inode = inode_ref (loc->inode);
+
+                local->call_cnt = layout->cnt;
+                call_cnt = local->call_cnt;
+
+                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
+                 *       revalidates directly go to the cached-subvolume.
+                 */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set dict value.");
+                        op_errno = -1;
+                        goto err;
+                }
+
+                for (i = 0; i < layout->cnt; i++) {
+                        subvol = layout->list[i].xlator;
+
+                        STACK_WIND (frame, dht_revalidate_cbk,
+                                    subvol, subvol->fops->lookup,
+                                    loc, local->xattr_req);
+
+                        if (!--call_cnt)
+                                break;
+                }
+        } else {
+        do_fresh_lookup:
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set dict value.");
+                        op_errno = -1;
+                        goto err;
+                }
+
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->link_xattr_name, 256);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set dict value.");
+                        op_errno = -1;
+                        goto err;
+                }
+
+                /* Send it to only local volume */
+                STACK_WIND (frame, nufa_local_lookup_cbk,
+                            (xlator_t *)conf->private,
+                            ((xlator_t *)conf->private)->fops->lookup,
+                            loc, local->xattr_req);
+        }
 
         return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL);
-	return 0;
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
+        return 0;
 }
 
 int
-nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie, 
-				 xlator_t *this, int op_ret, int op_errno,
-				 inode_t *inode, struct stat *stbuf)
+nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int op_ret, int op_errno,
+                                 inode_t *inode, struct iatt *stbuf,
+                                 struct iatt *preparent,
+                                 struct iatt *postparent, dict_t *xdata)
 {
- 	dht_local_t  *local = NULL;
- 	call_frame_t *prev = NULL;
-	dht_conf_t   *conf  = NULL;
-	
- 	local = frame->local;
- 	prev  = cookie;
- 	conf  = this->private;
-	
- 	if (op_ret == -1)
- 		goto err;
-	
- 	STACK_WIND (frame, dht_create_cbk,
- 		    conf->local_volume, conf->local_volume->fops->create,
- 		    &local->loc, local->flags, local->mode, local->fd);
-	
- 	return 0;
-	
- err:
- 	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL);	
- 	return 0;
+        dht_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1)
+                goto err;
+
+        STACK_WIND (frame, dht_create_cbk,
+                    local->cached_subvol, local->cached_subvol->fops->create,
+                    &local->loc, local->flags, local->mode, local->umask,
+                    local->fd, local->params);
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (create, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 int
 nufa_create (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, int32_t flags, mode_t mode, fd_t *fd)
+             loc_t *loc, int32_t flags, mode_t mode,
+             mode_t umask, fd_t *fd, dict_t *params)
 {
- 	dht_local_t *local = NULL;
-	dht_conf_t  *conf  = NULL;
-	xlator_t    *subvol = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
-
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
-
- 	conf  = this->private; 	
-
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
-
- 	if (subvol != conf->local_volume) {
- 		/* create a link file instead of actual file */
- 		ret = loc_copy (&local->loc, loc);
- 		if (ret == -1) {
- 			gf_log (this->name, GF_LOG_ERROR,
- 				"memory allocation failed :(");
- 			op_errno = ENOMEM;
- 			goto err;
- 		}
- 
- 		local->fd = fd_ref (fd);
- 		local->mode = mode;
- 		local->flags = flags;
- 		
- 		dht_linkfile_create (frame, nufa_create_linkfile_create_cbk,
- 				      conf->local_volume, subvol, loc);
- 		return 0;
- 	}
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"creating %s on %s", loc->path, subvol->name);
-
-	STACK_WIND (frame, dht_create_cbk,
-		    subvol, subvol->fops->create,
-		    loc, flags, mode, fd);
-
-	return 0;
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf  = NULL;
+        xlator_t    *subvol = NULL;
+        xlator_t    *avail_subvol = NULL;
+        int          op_errno = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        conf  = this->private;
+
+        dht_get_du_info (frame, this, loc);
+
+        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
+
+        avail_subvol = conf->private;
+        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
+                avail_subvol =
+                        dht_free_disk_available_subvol (this,
+                                                        (xlator_t *)conf->private,
+                                                        local);
+        }
+
+        if (subvol != avail_subvol) {
+                /* create a link file instead of actual file */
+                local->params = dict_ref (params);
+                local->mode = mode;
+                local->flags = flags;
+                local->umask = umask;
+                local->cached_subvol = avail_subvol;
+                dht_linkfile_create (frame, nufa_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
+                return 0;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "creating %s on %s", loc->path, subvol->name);
+
+        STACK_WIND (frame, dht_create_cbk,
+                    subvol, subvol->fops->create,
+                    loc, flags, mode, umask, fd, params);
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (create, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 int
 nufa_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int op_ret, int op_errno,
-			 inode_t *inode, struct stat *stbuf)
+                         int op_ret, int op_errno, inode_t *inode,
+                         struct iatt *stbuf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
- 	dht_local_t  *local = NULL;
- 	call_frame_t *prev = NULL;
-	dht_conf_t   *conf  = NULL;
-	
- 	local = frame->local;
- 	prev  = cookie;
- 	conf  = this->private;
- 	
- 	if (op_ret >= 0) {
- 		STACK_WIND (frame, dht_newfile_cbk,
- 			    conf->local_volume, 
- 			    conf->local_volume->fops->mknod,
- 			    &local->loc, local->mode, local->rdev);
-		
- 		return 0;
- 	}
-	
- 	DHT_STACK_UNWIND (frame, op_ret, op_errno, inode, stbuf);
- 	return 0;
-}
+        dht_local_t  *local = NULL;
 
+        local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
 
-int
-nufa_mknod (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, mode_t mode, dev_t rdev)
-{
- 	dht_local_t *local = NULL;
-	dht_conf_t  *conf  = NULL;
-	xlator_t    *subvol = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
-
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
-
- 	conf  = this->private; 	
-
-
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
-
-
- 	if (conf->local_volume != subvol) {
- 		/* Create linkfile first */
- 		ret = loc_copy (&local->loc, loc);
- 		if (ret == -1) {
- 			gf_log (this->name, GF_LOG_ERROR,
- 				"memory allocation failed :(");
- 			op_errno = ENOMEM;
- 			goto err;
- 		}
- 
-		local->mode = mode;
- 		local->rdev = rdev;
- 		
- 		dht_linkfile_create (frame, nufa_mknod_linkfile_cbk,
- 				      conf->local_volume, subvol, loc);
- 		return 0;
- 	}
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"creating %s on %s", loc->path, subvol->name);
-
-	STACK_WIND (frame, dht_newfile_cbk,
-		    subvol, subvol->fops->mknod,
-		    loc, mode, rdev);
-
-	return 0;
+        if (op_ret >= 0) {
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
+                            local->cached_subvol->fops->mknod,
+                            &local->loc, local->mode, local->rdev,
+                            local->umask, local->params);
 
+                return 0;
+        }
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+        WIPE (postparent);
+        WIPE (preparent);
 
-	return 0;
+        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
+                          inode, stbuf, preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
-notify (xlator_t *this, int event, void *data, ...)
+nufa_mknod (call_frame_t *frame, xlator_t *this,
+            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
 {
-	int ret = -1;
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf  = NULL;
+        xlator_t    *subvol = NULL;
+        xlator_t    *avail_subvol = NULL;
+        int          op_errno = -1;
 
-	ret = dht_notify (this, event, data);
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-	return ret;
-}
+        conf  = this->private;
 
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
+        dht_get_du_info (frame, this, loc);
 
-	conf = this->private;
+        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                FREE (conf->file_layouts[i]);
-                        }
-                        FREE (conf->file_layouts);
-                }
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
 
-                if (conf->default_dir_layout)
-                        FREE (conf->default_dir_layout);
+        /* Consider the disksize in consideration */
+        avail_subvol = conf->private;
+        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
+                avail_subvol =
+                        dht_free_disk_available_subvol (this,
+                                                        (xlator_t *)conf->private,
+                                                        local);
+        }
 
-                if (conf->subvolumes)
-                        FREE (conf->subvolumes);
+        if (avail_subvol != subvol) {
+                /* Create linkfile first */
 
-		if (conf->subvolume_status)
-			FREE (conf->subvolume_status);
+                local->params = dict_ref (params);
+                local->mode = mode;
+                local->umask = umask;
+                local->rdev = rdev;
+                local->cached_subvol = avail_subvol;
 
-                FREE (conf);
+                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk, this,
+                                     avail_subvol, subvol, loc);
+                return 0;
         }
 
-	return;
+        gf_log (this->name, GF_LOG_TRACE,
+                "creating %s on %s", loc->path, subvol->name);
+
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL);
+
+        return 0;
 }
 
-int
-init (xlator_t *this)
+
+gf_boolean_t
+same_first_part (char *str1, char term1, char *str2, char term2)
 {
-        dht_conf_t    *conf = NULL;
-	xlator_list_t *trav = NULL;
-	data_t        *data = NULL;
-	char          *local_volname = NULL;
-	char          *lookup_unhashed_str = NULL;
-        int            ret = -1;
-        int            i = 0;
-	char           my_hostname[256];
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"DHT needs more than one child defined");
-		return -1;
-	}
-  
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-        conf = CALLOC (1, sizeof (*conf));
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "memory allocation failed :(");
-                goto err;
+        gf_boolean_t    ended1;
+        gf_boolean_t    ended2;
+
+        for (;;) {
+                ended1 = ((*str1 == '\0') || (*str1 == term1));
+                ended2 = ((*str2 == '\0') || (*str2 == term2));
+                if (ended1 && ended2) {
+                        return _gf_true;
+                }
+                if (ended1 || ended2 || (*str1 != *str2)) {
+                        return _gf_false;
+                }
+                ++str1;
+                ++str2;
         }
+}
 
-	conf->search_unhashed = 0;
-
-	if (dict_get_str (this->options, "lookup-unhashed",
-			  &lookup_unhashed_str) == 0) {
-		gf_string2boolean (lookup_unhashed_str,
-				   &conf->search_unhashed);
-	}
+typedef struct nufa_args {
+        xlator_t    *this;
+        char        *volname;
+        gf_boolean_t addr_match;
+} nufa_args_t;
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
+static void
+nufa_find_local_brick (xlator_t *xl, void *data)
+{
+        nufa_args_t     *args = data;
+        xlator_t        *this = args->this;
+        char            *local_volname = args->volname;
+        gf_boolean_t    addr_match = args->addr_match;
+        char            *brick_host = NULL;
+        dht_conf_t      *conf = this->private;
+        int             ret = -1;
+
+        /*This means a local subvol was already found. We pick the first brick
+         * that is local*/
+        if (conf->private)
+                return;
+
+        if (strcmp (xl->name, local_volname) == 0) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using specified subvol %s",
+                        local_volname);
+                return;
         }
 
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
+        if (!addr_match)
+                return;
+
+        ret = dict_get_str (xl->options, "remote-host", &brick_host);
+        if ((ret == 0) &&
+            (gf_is_same_address (local_volname, brick_host) ||
+             gf_is_local_addr (brick_host))) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using the first local "
+                        "subvol %s", xl->name);
+                return;
         }
 
-	LOCK_INIT (&conf->subvolume_lock);
-
-	conf->gen = 1;
-
-	local_volname = "localhost";
-	ret = gethostname (my_hostname, 256);
-	if (ret < 0) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"could not find hostname (%s)",
-			strerror (errno));
-	}
-
-	if (ret == 0)
-		local_volname = my_hostname;
-
-	data = dict_get (this->options, "local-volume-name");
-	if (data) {
-		local_volname = data->data;
-	}
-
-	trav = this->children;
-	while (trav) {
-		if (strcmp (trav->xlator->name, local_volname) == 0)
-			break;
-		trav = trav->next;
-	}
-
-	if (!trav) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"Could not find subvolume named '%s'. "
-			"Please define volume with the name as the hostname "
-			"or override it with 'option local-volume-name'",
-			local_volname);
-		goto err;
-	}
-	/* The volume specified exists */
-	conf->local_volume = trav->xlator;
-
-        this->private = conf;
+}
 
-        return 0;
+static void
+nufa_to_dht (xlator_t *this)
+{
+        GF_ASSERT (this);
+        GF_ASSERT (this->fops);
 
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                FREE (conf->file_layouts[i]);
-                        }
-                        FREE (conf->file_layouts);
+        this->fops->lookup = dht_lookup;
+        this->fops->create = dht_create;
+        this->fops->mknod  = dht_mknod;
+}
+
+int
+nufa_find_local_subvol (xlator_t *this,
+                        void (*fn) (xlator_t *each, void* data), void *data)
+{
+        int             ret = -1;
+        dht_conf_t      *conf = this->private;
+        xlator_list_t   *trav = NULL;
+        xlator_t        *parent = NULL;
+        xlator_t        *candidate = NULL;
+
+        xlator_foreach_depth_first (this, fn, data);
+        if (!conf->private) {
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't find a local "
+                        "brick");
+                return -1;
+        }
+
+        candidate = conf->private;
+        trav = candidate->parents;
+        while (trav) {
+
+                parent = trav->xlator;
+                if (strcmp (parent->type, "cluster/nufa") == 0) {
+                        gf_log (this->name, GF_LOG_INFO, "Found local subvol, "
+                                "%s", candidate->name);
+                        ret = 0;
+                        conf->private = candidate;
+                        break;
                 }
 
-                if (conf->default_dir_layout)
-                        FREE (conf->default_dir_layout);
+                candidate = parent;
+                trav = parent->parents;
+        }
 
-                if (conf->subvolumes)
-                        FREE (conf->subvolumes);
+        return ret;
+}
 
-		if (conf->subvolume_status)
-			FREE (conf->subvolume_status);
+int
+nufa_init (xlator_t *this)
+{
+        data_t        *data = NULL;
+        char          *local_volname = NULL;
+        int            ret = -1;
+        char           my_hostname[256];
+        gf_boolean_t   addr_match = _gf_false;
+        nufa_args_t    args = {0, };
 
-                FREE (conf);
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
         }
 
-        return -1;
-}
+        if ((data = dict_get (this->options, "local-volume-name"))) {
+                local_volname = data->data;
 
+        } else {
+                addr_match = _gf_true;
+                local_volname = "localhost";
+                ret = gethostname (my_hostname, 256);
+                if (ret == 0)
+                        local_volname = my_hostname;
 
-struct xlator_fops fops = {
-	.lookup      = nufa_lookup,
-	.create      = nufa_create,
-	.mknod       = nufa_mknod,
-
-	.stat        = dht_stat,
-	.chmod       = dht_chmod,
-	.chown       = dht_chown,
-	.fchown      = dht_fchown,
-	.fchmod      = dht_fchmod,
-	.fstat       = dht_fstat,
-	.utimens     = dht_utimens,
-	.truncate    = dht_truncate,
-	.ftruncate   = dht_ftruncate,
-	.access      = dht_access,
-	.readlink    = dht_readlink,
-	.setxattr    = dht_setxattr,
-	.getxattr    = dht_getxattr,
-	.removexattr = dht_removexattr,
-	.open        = dht_open,
-	.readv       = dht_readv,
-	.writev      = dht_writev,
-	.flush       = dht_flush,
-	.fsync       = dht_fsync,
-	.statfs      = dht_statfs,
-	.lk          = dht_lk,
-	.opendir     = dht_opendir,
-	.readdir     = dht_readdir,
-	.fsyncdir    = dht_fsyncdir,
-	.symlink     = dht_symlink,
-	.unlink      = dht_unlink,
-	.link        = dht_link,
-	.mkdir       = dht_mkdir,
-	.rmdir       = dht_rmdir,
-	.rename      = dht_rename,
-	.inodelk     = dht_inodelk,
-	.finodelk    = dht_finodelk,
-	.entrylk     = dht_entrylk,
-	.fentrylk    = dht_fentrylk,
-	.xattrop     = dht_xattrop,
-	.fxattrop    = dht_fxattrop,
-#if 0
-	.setdents    = dht_setdents,
-	.getdents    = dht_getdents,
-	.checksum    = dht_checksum,
-#endif
-};
+                else
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "could not find hostname (%s)",
+                                strerror (errno));
 
+        }
+
+        args.this = this;
+        args.volname = local_volname;
+        args.addr_match = addr_match;
+        ret = nufa_find_local_subvol (this, nufa_find_local_brick, &args);
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Unable to find local subvolume, switching "
+                        "to dht mode");
+                nufa_to_dht (this);
+        }
+        return 0;
+}
 
-struct xlator_mops mops = {
+
+class_methods_t class_methods = {
+        .init           = nufa_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
 };
 
 
-struct xlator_cbks cbks = {
-//	.release    = dht_release,
-//      .releasedir = dht_releasedir,
-	.forget     = dht_forget
+struct xlator_fops fops = {
+        .lookup      = nufa_lookup,
+        .create      = nufa_create,
+        .mknod       = nufa_mknod,
+
+        .stat        = dht_stat,
+        .fstat       = dht_fstat,
+        .truncate    = dht_truncate,
+        .ftruncate   = dht_ftruncate,
+        .access      = dht_access,
+        .readlink    = dht_readlink,
+        .setxattr    = dht_setxattr,
+        .getxattr    = dht_getxattr,
+        .removexattr = dht_removexattr,
+        .open        = dht_open,
+        .readv       = dht_readv,
+        .writev      = dht_writev,
+        .flush       = dht_flush,
+        .fsync       = dht_fsync,
+        .statfs      = dht_statfs,
+        .lk          = dht_lk,
+        .opendir     = dht_opendir,
+        .readdir     = dht_readdir,
+        .readdirp    = dht_readdirp,
+        .fsyncdir    = dht_fsyncdir,
+        .symlink     = dht_symlink,
+        .unlink      = dht_unlink,
+        .link        = dht_link,
+        .mkdir       = dht_mkdir,
+        .rmdir       = dht_rmdir,
+        .rename      = dht_rename,
+        .inodelk     = dht_inodelk,
+        .finodelk    = dht_finodelk,
+        .entrylk     = dht_entrylk,
+        .fentrylk    = dht_fentrylk,
+        .xattrop     = dht_xattrop,
+        .fxattrop    = dht_fxattrop,
+        .setattr     = dht_setattr,
 };
 
 
-struct volume_options options[] = {
-	{ .key  = {"local-volume-name"}, 
-	  .type = GF_OPTION_TYPE_XLATOR 
-	},
-        { .key  = {"lookup-unhashed"}, 
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = {NULL} },
+struct xlator_cbks cbks = {
+        .forget     = dht_forget
 };
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
new file mode 100644
index 000000000..d3ea90ba8
--- /dev/null
+++ b/xlators/cluster/dht/src/switch.c
@@ -0,0 +1,904 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "dht-common.h"
+#include "dht-mem-types.h"
+
+#include <sys/time.h>
+#include <stdlib.h>
+#include <fnmatch.h>
+#include <string.h>
+
+extern struct volume_options options[];
+
+struct switch_sched_array {
+        xlator_t *xl;
+        int32_t   eligible;
+        int32_t   considered;
+};
+
+/* Select one of this struct based on the path's pattern match */
+struct switch_struct {
+        struct switch_struct      *next;
+        struct switch_sched_array *array;
+        int32_t                    node_index; /* Index of the node in
+                                                  this pattern. */
+        int32_t                    num_child;  /* Total num of child nodes
+                                                  with this pattern. */
+        char                       path_pattern[256];
+};
+
+/* TODO: all 'TODO's in dht.c holds good */
+/* This function should return child node as '*:subvolumes' is inserterd */
+
+static int32_t
+gf_switch_valid_child (xlator_t *this, const char *child)
+{
+        xlator_list_t *children = NULL;
+        int32_t        ret = 0;
+
+        children = this->children;
+        while (children) {
+                if (!strcmp (child, children->xlator->name)) {
+                        ret = 1;
+                        break;
+                }
+                children = children->next;
+        }
+
+        return ret;
+}
+
+static xlator_t *
+get_switch_matching_subvol (const char *path, dht_conf_t *conf,
+                            xlator_t *hashed_subvol)
+{
+        struct switch_struct *cond      = NULL;
+        struct switch_struct *trav      = NULL;
+        char                 *pathname  = NULL;
+        int                   idx       = 0;
+        xlator_t             *subvol    = NULL;
+
+        cond = conf->private;
+        subvol = hashed_subvol;
+        if (!cond)
+                goto out;
+
+        pathname = gf_strdup (path);
+        if (!pathname)
+                goto out;
+
+        trav = cond;
+        while (trav) {
+                if (fnmatch (trav->path_pattern,
+                             pathname, FNM_NOESCAPE) == 0) {
+                        for (idx = 0; idx < trav->num_child; idx++) {
+                                if (trav->array[idx].xl == hashed_subvol)
+                                        goto out;
+                        }
+                        idx = trav->node_index++;
+                        trav->node_index %= trav->num_child;
+                        subvol = trav->array[idx].xl;
+                        goto out;
+                }
+                trav = trav->next;
+        }
+out:
+        GF_FREE (pathname);
+
+        return subvol;
+}
+
+
+int
+switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno,
+                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                         struct iatt *postparent)
+{
+        xlator_t     *subvol      = NULL;
+        char          is_linkfile = 0;
+        char          is_dir      = 0;
+        dht_conf_t   *conf        = NULL;
+        dht_local_t  *local       = NULL;
+        loc_t        *loc         = NULL;
+        int           i           = 0;
+        call_frame_t *prev        = NULL;
+        int           call_cnt    = 0;
+        int           ret         = 0;
+
+        conf  = this->private;
+
+        prev  = cookie;
+        local = frame->local;
+        loc   = &local->loc;
+
+        if (ENTRY_MISSING (op_ret, op_errno)) {
+                if (conf->search_unhashed) {
+                        local->op_errno = ENOENT;
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
+                }
+        }
+
+        if (op_ret == -1)
+                goto out;
+
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
+        is_dir      = check_is_dir (inode, stbuf, xattr);
+
+        if (!is_dir && !is_linkfile) {
+                /* non-directory and not a linkfile */
+
+                ret = dht_layout_preset (this, prev->this, inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "could not set pre-set layout for subvol %s",
+                                prev->this->name);
+                        op_ret   = -1;
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
+                goto out;
+        }
+
+        if (is_dir) {
+                call_cnt        = conf->subvolume_cnt;
+                local->call_cnt = call_cnt;
+
+                local->inode = inode_ref (inode);
+                local->xattr = dict_ref (xattr);
+
+                local->op_ret = 0;
+                local->op_errno = 0;
+
+                local->layout = dht_layout_new (this, conf->subvolume_cnt);
+                if (!local->layout) {
+                        op_ret   = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "memory allocation failed :(");
+                        goto err;
+                }
+
+                for (i = 0; i < call_cnt; i++) {
+                        STACK_WIND (frame, dht_lookup_dir_cbk,
+                                    conf->subvolumes[i],
+                                    conf->subvolumes[i]->fops->lookup,
+                                    &local->loc, local->xattr_req);
+                }
+        }
+
+        if (is_linkfile) {
+                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+
+                if (!subvol) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "linkfile not having link subvolume. path=%s",
+                                loc->path);
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
+                }
+
+                STACK_WIND (frame, dht_lookup_linkfile_cbk,
+                            subvol, subvol->fops->lookup,
+                            &local->loc, local->xattr_req);
+        }
+
+        return 0;
+
+out:
+        if (!local->hashed_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        local->loc.path);
+                local->op_errno = ENOENT;
+                dht_lookup_everywhere (frame, this, loc);
+                return 0;
+        }
+
+        STACK_WIND (frame, dht_lookup_cbk,
+                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                    &local->loc, local->xattr_req);
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
+                          inode, stbuf, xattr, NULL);
+        return 0;
+}
+
+int
+switch_lookup (call_frame_t *frame, xlator_t *this,
+               loc_t *loc, dict_t *xattr_req)
+{
+        xlator_t     *hashed_subvol = NULL;
+        xlator_t     *cached_subvol = NULL;
+        xlator_t     *subvol = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
+        int           ret    = -1;
+        int           op_errno = -1;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        conf = this->private;
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        if (xattr_req) {
+                local->xattr_req = dict_ref (xattr_req);
+        } else {
+                local->xattr_req = dict_new ();
+        }
+
+        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+        cached_subvol = local->cached_subvol;
+
+        local->hashed_subvol = hashed_subvol;
+
+        if (is_revalidate (loc)) {
+                layout = local->layout;
+                if (!layout) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "revalidate without cache. path=%s",
+                                loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
+                if (layout->gen && (layout->gen < conf->gen)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "incomplete layout failure for path=%s",
+                                loc->path);
+                        dht_layout_unref (this, local->layout);
+                        goto do_fresh_lookup;
+                }
+
+                local->inode    = inode_ref (loc->inode);
+
+                local->call_cnt = layout->cnt;
+                call_cnt = local->call_cnt;
+
+                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
+                 * attribute, revalidates directly go to the cached-subvolume.
+                 */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
+
+                for (i = 0; i < layout->cnt; i++) {
+                        subvol = layout->list[i].xlator;
+
+                        STACK_WIND (frame, dht_revalidate_cbk,
+                                    subvol, subvol->fops->lookup,
+                                    loc, local->xattr_req);
+
+                        if (!--call_cnt)
+                                break;
+                }
+        } else {
+        do_fresh_lookup:
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
+
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->link_xattr_name, 256);
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set dict value for %s",
+                                conf->link_xattr_name);
+
+                if (!hashed_subvol) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "no subvolume in layout for path=%s, "
+                                "checking on all the subvols to see if "
+                                "it is a directory", loc->path);
+                        call_cnt        = conf->subvolume_cnt;
+                        local->call_cnt = call_cnt;
+
+                        local->layout = dht_layout_new (this,
+                                                        conf->subvolume_cnt);
+                        if (!local->layout) {
+                                op_errno = ENOMEM;
+                                goto err;
+                        }
+
+                        for (i = 0; i < call_cnt; i++) {
+                                STACK_WIND (frame, dht_lookup_dir_cbk,
+                                            conf->subvolumes[i],
+                                            conf->subvolumes[i]->fops->lookup,
+                                            &local->loc, local->xattr_req);
+                        }
+                        return 0;
+                }
+
+                /*  */
+                cached_subvol = get_switch_matching_subvol (loc->path, conf,
+                                                            hashed_subvol);
+                if (cached_subvol == hashed_subvol) {
+                        STACK_WIND (frame, dht_lookup_cbk,
+                                    hashed_subvol,
+                                    hashed_subvol->fops->lookup,
+                                    loc, local->xattr_req);
+                } else {
+                        STACK_WIND (frame, switch_local_lookup_cbk,
+                                    cached_subvol,
+                                    cached_subvol->fops->lookup,
+                                    loc, local->xattr_req);
+                }
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   inode_t *inode, struct iatt *stbuf,
+                                   struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1)
+                goto err;
+
+        STACK_WIND (frame, dht_create_cbk,
+                    local->cached_subvol, local->cached_subvol->fops->create,
+                    &local->loc, local->flags, local->mode, local->umask,
+                    local->fd, local->params);
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (create, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+switch_create (call_frame_t *frame, xlator_t *this,
+               loc_t *loc, int32_t flags, mode_t mode,
+               mode_t umask, fd_t *fd, dict_t *params)
+{
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf  = NULL;
+        xlator_t    *subvol = NULL;
+        xlator_t    *avail_subvol = NULL;
+        int          op_errno = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        conf  = this->private;
+
+        dht_get_du_info (frame, this, loc);
+
+        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
+
+        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
+        if (dht_is_subvol_filled (this, avail_subvol)) {
+                avail_subvol =
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
+        }
+
+        if (subvol != avail_subvol) {
+                /* create a link file instead of actual file */
+                local->mode = mode;
+                local->flags = flags;
+                local->umask = umask;
+                local->cached_subvol = avail_subvol;
+                dht_linkfile_create (frame, switch_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
+                return 0;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "creating %s on %s", loc->path, subvol->name);
+
+        STACK_WIND (frame, dht_create_cbk,
+                    subvol, subvol->fops->create,
+                    loc, flags, mode, umask, fd, params);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (create, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+switch_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, inode_t *inode,
+                           struct iatt *stbuf, struct iatt *preparent,
+                           struct iatt *postparent, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+
+        local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
+
+        if (op_ret >= 0) {
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
+                            local->cached_subvol->fops->mknod,
+                            &local->loc, local->mode, local->rdev,
+                            local->umask, local->params);
+
+                return 0;
+        }
+err:
+        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
+                          inode, stbuf, preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *params)
+{
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf  = NULL;
+        xlator_t    *subvol = NULL;
+        xlator_t    *avail_subvol = NULL;
+        int          op_errno = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        conf  = this->private;
+
+        dht_get_du_info (frame, this, loc);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
+
+        /* Consider the disksize in consideration */
+        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
+        if (dht_is_subvol_filled (this, avail_subvol)) {
+                avail_subvol =
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
+        }
+
+        if (avail_subvol != subvol) {
+                /* Create linkfile first */
+
+                local->params = dict_ref (params);
+                local->mode = mode;
+                local->umask = umask;
+                local->rdev = rdev;
+                local->cached_subvol = avail_subvol;
+
+                dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
+                                     this, avail_subvol, subvol, loc);
+                return 0;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "creating %s on %s", loc->path, subvol->name);
+
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+void
+switch_fini (xlator_t *this)
+{
+        dht_conf_t           *conf = NULL;
+        struct switch_struct *trav = NULL;
+        struct switch_struct *prev = NULL;
+
+        conf = this->private;
+
+        if (conf) {
+                trav = (struct switch_struct *)conf->private;
+                conf->private = NULL;
+                while (trav) {
+                        GF_FREE (trav->array);
+                        prev = trav;
+                        trav = trav->next;
+                        GF_FREE (prev);
+                }
+        }
+
+        dht_fini(this);
+}
+
+int
+set_switch_pattern (xlator_t *this, dht_conf_t *conf,
+                    const char *pattern_str)
+{
+        int                         flag = 0;
+        int                         idx = 0;
+        int                         index = 0;
+        int                         child_count = 0;
+        char                       *tmp = NULL;
+        char                       *tmp1 = NULL;
+        char                       *child = NULL;
+        char                       *tmp_str = NULL;
+        char                       *tmp_str1 = NULL;
+        char                       *dup_str = NULL;
+        char                       *dup_childs = NULL;
+        char                       *switch_str = NULL;
+        char                       *pattern = NULL;
+        char                       *childs = NULL;
+        char                       *option_string = NULL;
+        struct switch_struct        *switch_buf = NULL;
+        struct switch_struct        *switch_opt = NULL;
+        struct switch_struct        *trav = NULL;
+        struct switch_sched_array  *switch_buf_array = NULL;
+        xlator_list_t              *trav_xl = NULL;
+
+        trav_xl = this->children;
+        while (trav_xl) {
+                index++;
+                trav_xl = trav_xl->next;
+        }
+        child_count = index;
+        switch_buf_array = GF_CALLOC ((index + 1),
+                                      sizeof (struct switch_sched_array),
+                                      gf_switch_mt_switch_sched_array);
+        if (!switch_buf_array)
+                goto err;
+
+        trav_xl = this->children;
+        index = 0;
+
+        while (trav_xl) {
+                switch_buf_array[index].xl = trav_xl->xlator;
+                switch_buf_array[index].eligible = 1;
+                trav_xl = trav_xl->next;
+                index++;
+        }
+
+        /*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
+
+        /* Get the pattern for considering switch case.
+           "option block-size *avi:10MB" etc */
+        option_string = gf_strdup (pattern_str);
+        switch_str = strtok_r (option_string, ";", &tmp_str);
+        while (switch_str) {
+                dup_str = gf_strdup (switch_str);
+                switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
+                                        gf_switch_mt_switch_struct);
+                if (!switch_opt) {
+                        GF_FREE (dup_str);
+                        goto err;
+                }
+
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                childs = strtok_r (NULL, ":", &tmp_str1);
+                if (strncmp (pattern, "*", 2) == 0) {
+                        gf_log ("switch", GF_LOG_INFO,
+                                "'*' pattern will be taken by default "
+                                "for all the unconfigured child nodes,"
+                                " hence neglecting current option");
+                        switch_str = strtok_r (NULL, ";", &tmp_str);
+                        GF_FREE (switch_opt);
+                        GF_FREE (dup_str);
+                        continue;
+                }
+                memcpy (switch_opt->path_pattern, pattern, strlen (pattern));
+                if (childs) {
+                        dup_childs = gf_strdup (childs);
+                        child = strtok_r (dup_childs, ",", &tmp);
+                        while (child) {
+                                if (gf_switch_valid_child (this, child)) {
+                                        idx++;
+                                        child = strtok_r (NULL, ",", &tmp);
+                                } else {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "%s is not a subvolume of %s. "
+                                                "pattern can only be scheduled "
+                                                "only to a subvolume of %s",
+                                                child, this->name, this->name);
+                                        goto err;
+                                }
+                        }
+                        GF_FREE (dup_childs);
+                        child = strtok_r (childs, ",", &tmp1);
+                        switch_opt->num_child = idx;
+                        switch_opt->array = GF_CALLOC (1, (idx *
+                                                           sizeof (struct switch_sched_array)),
+                                                       gf_switch_mt_switch_sched_array);
+                        if (!switch_opt->array)
+                                goto err;
+                        idx = 0;
+                        while (child) {
+                                for (index = 0; index < child_count; index++) {
+                                        if (strcmp (switch_buf_array[index].xl->name,
+                                                    child) == 0) {
+                                                gf_log ("switch", GF_LOG_DEBUG,
+                                                        "'%s' pattern will be "
+                                                        "scheduled to \"%s\"",
+                                                        switch_opt->path_pattern, child);
+                                                /*
+                                                  if (switch_buf_array[index-1].considered) {
+                                                  gf_log ("switch", GF_LOG_DEBUG,
+                                                  "ambiguity found, exiting");
+                                                  return -1;
+                                                  }
+                                                */
+                                                switch_opt->array[idx].xl = switch_buf_array[index].xl;
+                                                switch_buf_array[index].considered = 1;
+                                                idx++;
+                                                break;
+                                        }
+                                }
+                                child = strtok_r (NULL, ",", &tmp1);
+                        }
+                } else {
+                        /* error */
+                        gf_log ("switch", GF_LOG_ERROR,
+                                "Check \"scheduler.switch.case\" "
+                                "option in unify volume. Exiting");
+                        goto err;
+                }
+                GF_FREE (dup_str);
+
+                /* Link it to the main structure */
+                if (switch_buf) {
+                        /* there are already few entries */
+                        trav = switch_buf;
+                        while (trav->next)
+                                trav = trav->next;
+                        trav->next = switch_opt;
+                } else {
+                        /* First entry */
+                        switch_buf = switch_opt;
+                }
+                switch_opt = NULL;
+                switch_str = strtok_r (NULL, ";", &tmp_str);
+        }
+
+        /* Now, all the pattern based considerations done, so for all the
+         * remaining pattern, '*' to all the remaining child nodes
+         */
+        {
+                for (index=0; index < child_count; index++) {
+                        /* check for considered flag */
+                        if (switch_buf_array[index].considered)
+                                continue;
+                        flag++;
+                }
+                if (!flag) {
+                        gf_log ("switch", GF_LOG_ERROR,
+                                "No nodes left for pattern '*'. Exiting");
+                        goto err;
+                }
+                switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
+                                        gf_switch_mt_switch_struct);
+                if (!switch_opt)
+                        goto err;
+
+                /* Add the '*' pattern to the array */
+                memcpy (switch_opt->path_pattern, "*", 2);
+                switch_opt->num_child = flag;
+                switch_opt->array =
+                        GF_CALLOC (1,
+                                   flag * sizeof (struct switch_sched_array),
+                                   gf_switch_mt_switch_sched_array);
+                if (!switch_opt->array)
+                        goto err;
+                flag = 0;
+                for (index=0; index < child_count; index++) {
+                        /* check for considered flag */
+                        if (switch_buf_array[index].considered)
+                                continue;
+                        gf_log ("switch", GF_LOG_DEBUG,
+                                "'%s' pattern will be scheduled to \"%s\"",
+                                switch_opt->path_pattern,
+                                switch_buf_array[index].xl->name);
+                        switch_opt->array[flag].xl =
+                                switch_buf_array[index].xl;
+                        switch_buf_array[index].considered = 1;
+                        flag++;
+                }
+                if (switch_buf) {
+                        /* there are already few entries */
+                        trav = switch_buf;
+                        while (trav->next)
+                                trav = trav->next;
+                        trav->next = switch_opt;
+                } else {
+                        /* First entry */
+                        switch_buf = switch_opt;
+                }
+                switch_opt = NULL;
+        }
+        /* */
+        conf->private = switch_buf;
+
+        return 0;
+err:
+        GF_FREE (switch_buf_array);
+        GF_FREE (switch_opt);
+
+        if (switch_buf) {
+                trav = switch_buf;
+                while (trav) {
+                        GF_FREE (trav->array);
+                        switch_opt = trav;
+                        trav = trav->next;
+                        GF_FREE (switch_opt);
+                }
+        }
+        return -1;
+}
+
+
+int32_t
+switch_init (xlator_t *this)
+{
+        dht_conf_t            *conf = NULL;
+        data_t                *data = NULL;
+        int                    ret = -1;
+
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
+        }
+        conf = this->private;
+
+        data = dict_get (this->options, "pattern.switch.case");
+        if (data) {
+                /* TODO: */
+                ret = set_switch_pattern (this, conf, data->data);
+                if (ret) {
+                        goto err;
+                }
+        }
+
+        this->private = conf;
+        return 0;
+
+err:
+        dht_fini(this);
+        return -1;
+}
+
+
+class_methods_t class_methods = {
+        .init           = switch_init,
+        .fini           = switch_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
+
+
+struct xlator_fops fops = {
+        .lookup      = switch_lookup,
+        .create      = switch_create,
+        .mknod       = switch_mknod,
+
+        .stat        = dht_stat,
+        .fstat       = dht_fstat,
+        .truncate    = dht_truncate,
+        .ftruncate   = dht_ftruncate,
+        .access      = dht_access,
+        .readlink    = dht_readlink,
+        .setxattr    = dht_setxattr,
+        .getxattr    = dht_getxattr,
+        .removexattr = dht_removexattr,
+        .open        = dht_open,
+        .readv       = dht_readv,
+        .writev      = dht_writev,
+        .flush       = dht_flush,
+        .fsync       = dht_fsync,
+        .statfs      = dht_statfs,
+        .lk          = dht_lk,
+        .opendir     = dht_opendir,
+        .readdir     = dht_readdir,
+        .readdirp    = dht_readdirp,
+        .fsyncdir    = dht_fsyncdir,
+        .symlink     = dht_symlink,
+        .unlink      = dht_unlink,
+        .link        = dht_link,
+        .mkdir       = dht_mkdir,
+        .rmdir       = dht_rmdir,
+        .rename      = dht_rename,
+        .inodelk     = dht_inodelk,
+        .finodelk    = dht_finodelk,
+        .entrylk     = dht_entrylk,
+        .fentrylk    = dht_fentrylk,
+        .xattrop     = dht_xattrop,
+        .fxattrop    = dht_fxattrop,
+        .setattr     = dht_setattr,
+};
+
+
+struct xlator_cbks cbks = {
+        .forget     = dht_forget
+};
diff --git a/xlators/cluster/ha/src/Makefile.am b/xlators/cluster/ha/src/Makefile.am
index 069a0dcde..5c1364b7f 100644
--- a/xlators/cluster/ha/src/Makefile.am
+++ b/xlators/cluster/ha/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = ha.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-ha_la_LDFLAGS = -module -avoidversion 
+ha_la_LDFLAGS = -module -avoid-version 
 
 ha_la_SOURCES = ha-helpers.c ha.c
 ha_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = ha.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/ha/src/ha-helpers.c b/xlators/cluster/ha/src/ha-helpers.c
index a88adad97..19be1ed27 100644
--- a/xlators/cluster/ha/src/ha-helpers.c
+++ b/xlators/cluster/ha/src/ha-helpers.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include "xlator.h"
 #include "call-stub.h"
 #include "defaults.h"
@@ -49,12 +39,14 @@ int ha_alloc_init_fd (call_frame_t *frame, fd_t *fd)
 			goto out;
 		}
 		hafdp = (hafd_t *)(long)tmp_hafdp;
-		local = frame->local = CALLOC (1, sizeof (*local));
+                local = frame->local = GF_CALLOC (1, sizeof (*local), 
+                                                  gf_ha_mt_ha_local_t);
 		if (local == NULL) {
 			ret = -ENOMEM;
 			goto out;
 		}
-		local->state = CALLOC (1, child_count);
+                local->state = GF_CALLOC (1, child_count, 
+                                          gf_ha_mt_child_count);
 		if (local->state == NULL) {
 			ret = -ENOMEM;
 			goto out;
@@ -141,11 +133,17 @@ int ha_handle_cbk (call_frame_t *frame, void *cookie, int op_ret, int op_errno)
 			}
 		}
 	}
-	if (local->stub)
+	if (local->stub) {
 		call_stub_destroy (local->stub);
+                local->stub = NULL;
+        }
+
 	if (local->fd) {
-		FREE (local->state);
+		GF_FREE (local->state);
+                local->state = NULL;
+
 		fd_unref (local->fd);
+                local->fd = NULL;
 	}
 	return 0;
 }
@@ -164,7 +162,8 @@ int ha_alloc_init_inode (call_frame_t *frame, inode_t *inode)
 	local = frame->local;
 
 	if (local == NULL) {
-		local = frame->local = CALLOC (1, sizeof (*local));
+                local = frame->local = GF_CALLOC (1, sizeof (*local), 
+                                                  gf_ha_mt_ha_local_t);
 		if (local == NULL) {
 			ret = -ENOMEM;
 			goto out;
diff --git a/xlators/cluster/ha/src/ha-mem-types.h b/xlators/cluster/ha/src/ha-mem-types.h
new file mode 100644
index 000000000..e5e97d237
--- /dev/null
+++ b/xlators/cluster/ha/src/ha-mem-types.h
@@ -0,0 +1,26 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __HA_MEM_TYPES_H__
+#define __HA_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_ha_mem_types_ {
+        gf_ha_mt_ha_local_t = gf_common_mt_end + 1,
+        gf_ha_mt_hafd_t,
+        gf_ha_mt_char,
+        gf_ha_mt_child_count,
+        gf_ha_mt_xlator_t,
+        gf_ha_mt_ha_private_t,
+        gf_ha_mt_end
+};
+#endif
+
diff --git a/xlators/cluster/ha/src/ha.c b/xlators/cluster/ha/src/ha.c
index 4542bdc7e..3eccb516b 100644
--- a/xlators/cluster/ha/src/ha.c
+++ b/xlators/cluster/ha/src/ha.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 /* generate errors randomly, code is simple now, better alogorithm
  * can be written to decide what error to be returned and when
  */
@@ -41,6 +31,41 @@
  * - do not alloc the call-stub in case only one subvol is up.
  */
 
+void
+ha_local_wipe (ha_local_t *local)
+{
+        if (local->stub) {
+                call_stub_destroy (local->stub);
+                local->stub = NULL;
+        }
+
+        if (local->state) {
+                GF_FREE (local->state);
+                local->state = NULL;
+        }
+
+        if (local->dict) {
+                dict_unref (local->dict);
+                local->dict = NULL;
+        }
+
+        loc_wipe (&local->loc);
+
+        if (local->fd) {
+                fd_unref (local->fd);
+                local->fd = NULL;
+        }
+
+        if (local->inode) {
+                inode_unref (local->inode);
+                local->inode = NULL;
+        }
+
+        GF_FREE (local);
+        return;
+}
+
+
 int
 ha_forget (xlator_t *this,
 	   inode_t *inode)
@@ -49,7 +74,7 @@ ha_forget (xlator_t *this,
 	char *state = NULL;
 	if (!inode_ctx_del (inode, this, &stateino)) {
 		state =  ((char *)(long)stateino);
-		FREE (state);
+		GF_FREE (state);
 	}
 
 	return 0;
@@ -63,8 +88,9 @@ ha_lookup_cbk (call_frame_t *frame,
 	       int32_t op_ret,
 	       int32_t op_errno,
 	       inode_t *inode,
-	       struct stat *buf,
-	       dict_t *dict)
+	       struct iatt *buf,
+               dict_t *dict,
+               struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -106,6 +132,7 @@ ha_lookup_cbk (call_frame_t *frame,
 	if (local->op_ret == -1 && op_ret == 0) {
 		local->op_ret = 0;
 		local->buf = *buf;
+                local->postparent = *postparent;
 		if (dict)
 			local->dict = dict_ref (dict);
 	}
@@ -127,7 +154,8 @@ ha_lookup_cbk (call_frame_t *frame,
 			      local->op_errno,
 			      inode,
 			      &local->buf,
-			      ctx);
+			      ctx,
+                              &local->postparent);
 		if (inode)
 			inode_unref (inode);
 		if (ctx)
@@ -148,19 +176,33 @@ ha_lookup (call_frame_t *frame,
 	char *state = NULL;
 	xlator_t **children = NULL;
 	int ret = -1;
+        int32_t op_errno = EINVAL;
 
 	local = frame->local;
 	pvt = this->private;
 	child_count = pvt->child_count;
 	children = pvt->children;
 
-	frame->local = local = CALLOC (1, sizeof (*local));
+	frame->local = local = GF_CALLOC (1, sizeof (*local), 
+                                          gf_ha_mt_ha_local_t);
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
 	child_count = pvt->child_count;
 	local->inode = inode_ref (loc->inode);
 
 	ret = inode_ctx_get (loc->inode, this, NULL);
 	if (ret) {
-		state = CALLOC (1, child_count);
+		state = GF_CALLOC (1, child_count, gf_ha_mt_child_count);
+                if (state == NULL) {
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+
 		inode_ctx_put (loc->inode, this, (uint64_t)(long)state);
 	} else
 		local->revalidate = 1;
@@ -178,6 +220,14 @@ ha_lookup (call_frame_t *frame,
 			    xattr_req);
 	}
 	return 0;
+
+unwind:
+        local = frame->local;
+        frame->local = NULL;
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
+
+        ha_local_wipe (local);
+        return 0;
 }
 
  int32_t
@@ -186,7 +236,7 @@ ha_stat_cbk (call_frame_t *frame,
 	     xlator_t *this,
 	     int32_t op_ret,
 	     int32_t op_errno,
-	     struct stat *buf)
+	     struct iatt *buf)
 {
 	int ret = -1;
 
@@ -229,135 +279,25 @@ err:
 	return 0;	
 }
 
- int32_t
-ha_chmod_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct stat *buf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
-	}
-	return 0;
-}
-
 int32_t
-ha_chmod (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc,
-	  mode_t mode)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_chmod_stub (frame, ha_chmod, loc, mode);
-
-	STACK_WIND_COOKIE (frame,
-			   ha_chmod_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->chmod,
-			   loc,
-			   mode);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-	return 0;
-}
-
- int32_t
-ha_fchmod_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct stat *buf)
+ha_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                struct iatt *statpost)
 {
 	int ret = -1;
 
 	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
 
 	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
+		STACK_UNWIND (frame, op_ret, op_errno, statpre, statpost);
 	}
 	return 0;
 }
 
-int32_t 
-ha_fchmod (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd,
-	   mode_t mode)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fchmod_stub (frame, ha_fchmod, fd, mode);
-
-	STACK_WIND_COOKIE (frame,
-			   ha_fchmod_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fchmod,
-			   fd,
-			   mode);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-	return 0;
-}
-
- int32_t
-ha_chown_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct stat *buf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
-	}
-	return 0;
-}
 
 int32_t
-ha_chown (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc,
-	  uid_t uid,
-	  gid_t gid)
+ha_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid)
 {
 	ha_local_t *local = NULL;
 	int op_errno = 0;
@@ -368,49 +308,24 @@ ha_chown (call_frame_t *frame,
 		goto err;
 	}
 	local = frame->local;
-	local->stub = fop_chown_stub (frame, ha_chown, loc, uid, gid);
+	local->stub = fop_setattr_stub (frame, ha_setattr, loc, stbuf, valid);
 
-	STACK_WIND_COOKIE (frame,	      
-			   ha_chown_cbk,
+	STACK_WIND_COOKIE (frame,
+			   ha_setattr_cbk,
 			   (void *)(long)local->active,
 			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->chown,
-			   loc,
-			   uid,
-			   gid);
+			   HA_ACTIVE_CHILD(this, local)->fops->setattr,
+			   loc, stbuf, valid);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, ENOTCONN, NULL);
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
- int32_t
-ha_fchown_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct stat *buf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
 
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
-	}
-	return 0;
-}
-
-int32_t 
-ha_fchown (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd,
-	   uid_t uid,
-	   gid_t gid)
+int32_t
+ha_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid)
 {
 	ha_local_t *local = NULL;
 	int op_errno = 0;
@@ -421,29 +336,29 @@ ha_fchown (call_frame_t *frame,
 		goto err;
 	}
 	local = frame->local;
-	local->stub = fop_fchown_stub (frame, ha_fchown, fd, uid, gid);
+	local->stub = fop_fsetattr_stub (frame, ha_fsetattr, fd, stbuf, valid);
 
 	STACK_WIND_COOKIE (frame,	      
-			   ha_fchown_cbk,
+			   ha_setattr_cbk,
 			   (void *)(long)local->active,
 			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fchown,
-			   fd,
-			   uid,
-			   gid);
+			   HA_ACTIVE_CHILD(this, local)->fops->fsetattr,
+			   fd, stbuf, valid);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
- int32_t
+
+int32_t
 ha_truncate_cbk (call_frame_t *frame,
 		 void *cookie,
 		 xlator_t *this,
 		 int32_t op_ret,
 		 int32_t op_errno,
-		 struct stat *buf)
+		 struct iatt *prebuf,
+                 struct iatt *postbuf)
 {
 	int ret = -1;
 
@@ -452,7 +367,8 @@ ha_truncate_cbk (call_frame_t *frame,
 		STACK_UNWIND (frame,
 			      op_ret,
 			      op_errno,
-			      buf);
+			      prebuf,
+                              postbuf);
 	}
 	return 0;
 }
@@ -473,6 +389,11 @@ ha_truncate (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_truncate_stub (frame, ha_truncate, loc, offset);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_truncate_cbk,
@@ -483,7 +404,7 @@ ha_truncate (call_frame_t *frame,
 			   offset);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
@@ -493,7 +414,8 @@ ha_ftruncate_cbk (call_frame_t *frame,
 		  xlator_t *this,
 		  int32_t op_ret,
 		  int32_t op_errno,
-		  struct stat *buf)
+		  struct iatt *prebuf,
+                  struct iatt *postbuf)
 {
 	int ret = -1;
 
@@ -503,7 +425,7 @@ ha_ftruncate_cbk (call_frame_t *frame,
 		STACK_UNWIND (frame,
 			      op_ret,
 			      op_errno,
-			      buf);
+			      prebuf, postbuf);
 	}
 	return 0;
 }
@@ -524,6 +446,11 @@ ha_ftruncate (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_ftruncate_stub (frame, ha_ftruncate, fd, offset);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_ftruncate_cbk,
@@ -534,58 +461,12 @@ ha_ftruncate (call_frame_t *frame,
 			   offset);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-	return 0;
-}
-
-int32_t 
-ha_utimens_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		struct stat *buf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
-	}
-	return 0;
-}
+        local = frame->local;
+        frame->local = NULL;
 
-int32_t 
-ha_utimens (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    struct timespec tv[2])
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_utimens_stub (frame, ha_utimens, loc, tv);
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
 
-	STACK_WIND_COOKIE (frame,
-			   ha_utimens_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->utimens,
-			   loc,
-			   tv);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -624,6 +505,11 @@ ha_access (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_access_stub (frame, ha_access, loc, mask);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_access_cbk,
@@ -634,7 +520,12 @@ ha_access (call_frame_t *frame,
 			   mask);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -645,7 +536,8 @@ ha_readlink_cbk (call_frame_t *frame,
 		 xlator_t *this,
 		 int32_t op_ret,
 		 int32_t op_errno,
-		 const char *path)
+		 const char *path,
+                 struct iatt *sbuf)
 {
 	int ret = -1;
 
@@ -655,7 +547,8 @@ ha_readlink_cbk (call_frame_t *frame,
 		STACK_UNWIND (frame,
 			      op_ret,
 			      op_errno,
-			      path);
+			      path,
+                              sbuf);
 	}
 	return 0;
 }
@@ -686,7 +579,7 @@ ha_readlink (call_frame_t *frame,
 			   size);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
@@ -697,8 +590,9 @@ ha_mknod_lookup_cbk (call_frame_t *frame,
 		     int32_t op_ret,
 		     int32_t op_errno,
 		     inode_t *inode,
-		     struct stat *buf,
-		     dict_t *dict)
+		     struct iatt *buf,
+                     dict_t *dict,
+                     struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -742,12 +636,13 @@ ha_mknod_lookup_cbk (call_frame_t *frame,
 
 	if (cnt == 0) {
 		call_stub_t *stub = local->stub;
-		FREE (local->state);
+		GF_FREE (local->state);
 		STACK_UNWIND (frame,
 			      local->op_ret,
 			      local->op_errno,
 			      local->stub->args.mknod.loc.inode,
-			      &local->buf);
+			      &local->buf, &local->preparent,
+                              &local->postparent);
 		call_stub_destroy (stub);
 	}
 	return 0;
@@ -760,7 +655,9 @@ ha_mknod_cbk (call_frame_t *frame,
 	      int32_t op_ret,
 	      int32_t op_errno,
 	      inode_t *inode,
-	      struct stat *buf)
+              struct iatt *buf,
+              struct iatt *preparent,
+              struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -798,6 +695,8 @@ ha_mknod_cbk (call_frame_t *frame,
 		local->op_ret = 0;
 		local->first_success = 1;
 		local->buf = *buf;
+                local->preparent = *preparent;
+                local->postparent = *postparent;
 	}
 	cnt = --local->call_count;
 	for (i = local->active + 1; i < child_count; i++) {
@@ -807,9 +706,11 @@ ha_mknod_cbk (call_frame_t *frame,
 
 	if (cnt == 0 || i == child_count) {
 		call_stub_t *stub = local->stub;
-		FREE (local->state);
+		GF_FREE (local->state);
 		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, local->stub->args.mknod.loc.inode, &local->buf);
+		STACK_UNWIND (frame, local->op_ret, local->op_errno,
+                              local->stub->args.mknod.loc.inode, &local->buf,
+                              &local->preparent, &local->postparent);
 		call_stub_destroy (stub);
 		return 0;
 	}
@@ -854,20 +755,46 @@ ha_mknod (call_frame_t *frame,
 	ha_private_t *pvt = NULL;
 	int child_count = 0, i = 0;
 	char *stateino = NULL;
+        int32_t op_errno = EINVAL;
 
 	local = frame->local;
 	pvt = this->private;
 	child_count = pvt->child_count;
 
-	frame->local = local = CALLOC (1, sizeof (*local));
+	frame->local = local = GF_CALLOC (1, sizeof (*local), 
+                                          gf_ha_mt_ha_local_t);
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	local->stub = fop_mknod_stub (frame, ha_mknod, loc, mode, rdev);
+        if (!local->stub) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	local->op_ret = -1;
 	local->op_errno = ENOTCONN;
-	local->state = CALLOC (1, child_count);
+	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!local->state) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	memcpy (local->state, pvt->state, child_count);
 	local->active = -1;
 
-	stateino = CALLOC (1, child_count);
+	stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!stateino) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
 
 	for (i = 0; i < child_count; i++) {
@@ -884,6 +811,14 @@ ha_mknod (call_frame_t *frame,
 		    HA_ACTIVE_CHILD(this, local)->fops->mknod,
 		    loc, mode, rdev);
 	return 0;
+
+err:
+        local = frame->local;
+        frame->local = NULL;
+
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        ha_local_wipe (local);
+        return 0;
 }
 
 
@@ -894,8 +829,9 @@ ha_mkdir_lookup_cbk (call_frame_t *frame,
 		     int32_t op_ret,
 		     int32_t op_errno,
 		     inode_t *inode,
-		     struct stat *buf,
-		     dict_t *dict)
+		     struct iatt *buf,
+                     dict_t *dict,
+                     struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -931,12 +867,12 @@ ha_mkdir_lookup_cbk (call_frame_t *frame,
 
 	if (cnt == 0) {
 		call_stub_t *stub = local->stub;
-		FREE (local->state);
+		GF_FREE (local->state);
 		STACK_UNWIND (frame,
 			      local->op_ret,
 			      local->op_errno,
-			      local->stub->args.mkdir.loc.inode,
-			      &local->buf);
+			      local->stub->args.mkdir.loc.inode, &local->buf,
+			      &local->preparent, &local->postparent);
 		call_stub_destroy (stub);
 	}
 	return 0;
@@ -949,7 +885,9 @@ ha_mkdir_cbk (call_frame_t *frame,
 	      int32_t op_ret,
 	      int32_t op_errno,
 	      inode_t *inode,
-	      struct stat *buf)
+              struct iatt *buf,
+              struct iatt *preparent,
+              struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -983,6 +921,8 @@ ha_mkdir_cbk (call_frame_t *frame,
 		local->op_ret = 0;
 		local->first_success = 1;
 		local->buf = *buf;
+                local->preparent = *preparent;
+                local->postparent = *postparent;
 	}
 	cnt = --local->call_count;
 	for (i = local->active + 1; i < child_count; i++) {
@@ -992,9 +932,11 @@ ha_mkdir_cbk (call_frame_t *frame,
 
 	if (cnt == 0 || i == child_count) {
 		call_stub_t *stub = local->stub;
-		FREE (local->state);
+		GF_FREE (local->state);
 		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, local->stub->args.mkdir.loc.inode, &local->buf);
+		STACK_UNWIND (frame, local->op_ret, local->op_errno,
+                              local->stub->args.mkdir.loc.inode, &local->buf,
+                              &local->preparent, &local->postparent);
 		call_stub_destroy (stub);
 		return 0;
 	}
@@ -1037,20 +979,46 @@ ha_mkdir (call_frame_t *frame,
 	ha_private_t *pvt = NULL;
 	int child_count = 0, i = 0;
 	char *stateino = NULL;
+        int32_t op_errno = EINVAL;
 
 	local = frame->local;
 	pvt = this->private;
 	child_count = pvt->child_count;
 
-	frame->local = local = CALLOC (1, sizeof (*local));
+	frame->local = local = GF_CALLOC (1, sizeof (*local), 
+                                          gf_ha_mt_ha_local_t);
+        if (!frame->local) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	local->stub = fop_mkdir_stub (frame, ha_mkdir, loc, mode);
+        if (!local->stub) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	local->op_ret = -1;
 	local->op_errno = ENOTCONN;
-	local->state = CALLOC (1, child_count);
+	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!local->state) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	memcpy (local->state, pvt->state, child_count);
 	local->active = -1;
 
-	stateino = CALLOC (1, child_count);
+	stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!stateino) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
 	for (i = 0; i < child_count; i++) {
 		if (local->state[i]) {
@@ -1066,6 +1034,13 @@ ha_mkdir (call_frame_t *frame,
 		    HA_ACTIVE_CHILD(this, local)->fops->mkdir,
 		    loc, mode);
 	return 0;
+err:
+        local = frame->local;
+        frame->local = NULL;
+
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        ha_local_wipe (local);
+        return 0;
 }
 
  int32_t
@@ -1073,13 +1048,15 @@ ha_unlink_cbk (call_frame_t *frame,
 	       void *cookie,
 	       xlator_t *this,
 	       int32_t op_ret,
-	       int32_t op_errno)
+	       int32_t op_errno,
+               struct iatt *preparent,
+               struct iatt *postparent)
 {
 	int ret = -1;
 
 	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
 	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno);
+		STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
 	}
 	return 0;
 }
@@ -1100,6 +1077,11 @@ ha_unlink (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_unlink_stub (frame, ha_unlink, loc);
+        if (!local->stub) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_unlink_cbk,
@@ -1109,7 +1091,7 @@ ha_unlink (call_frame_t *frame,
 			   loc);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno);
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
@@ -1118,7 +1100,9 @@ ha_rmdir_cbk (call_frame_t *frame,
 	      void *cookie,
 	      xlator_t *this,
 	      int32_t op_ret,
-	      int32_t op_errno)
+	      int32_t op_errno,
+              struct iatt *preparent,
+              struct iatt *postparent)
 {
 	int ret = -1;
 
@@ -1127,7 +1111,9 @@ ha_rmdir_cbk (call_frame_t *frame,
 	if (ret == 0) {
 		STACK_UNWIND (frame,
 			      op_ret,
-			      op_errno);
+			      op_errno,
+                              preparent,
+                              postparent);
 	}
 	return 0;
 }
@@ -1147,6 +1133,11 @@ ha_rmdir (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_rmdir_stub (frame, ha_rmdir, loc);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_rmdir_cbk,
@@ -1156,7 +1147,7 @@ ha_rmdir (call_frame_t *frame,
 			   loc);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno);
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
@@ -1168,8 +1159,9 @@ ha_symlink_lookup_cbk (call_frame_t *frame,
 		       int32_t op_ret,
 		       int32_t op_errno,
 		       inode_t *inode,
-		       struct stat *buf,
-		       dict_t *dict)
+		       struct iatt *buf,
+                       dict_t *dict,
+                       struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -1205,12 +1197,12 @@ ha_symlink_lookup_cbk (call_frame_t *frame,
 
 	if (cnt == 0) {
 		call_stub_t *stub = local->stub;
-		FREE (local->state);
+		GF_FREE (local->state);
 		STACK_UNWIND (frame,
 			      local->op_ret,
 			      local->op_errno,
-			      local->stub->args.symlink.loc.inode,
-			      &local->buf);
+			      local->stub->args.symlink.loc.inode, &local->buf,
+                              &local->preparent, &local->postparent);
 		call_stub_destroy (stub);
 	}
 	return 0;
@@ -1223,7 +1215,9 @@ ha_symlink_cbk (call_frame_t *frame,
 		int32_t op_ret,
 		int32_t op_errno,
 		inode_t *inode,
-		struct stat *buf)
+                struct iatt *buf,
+                struct iatt *preparent,
+                struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -1256,6 +1250,8 @@ ha_symlink_cbk (call_frame_t *frame,
 		local->op_ret = 0;
 		local->first_success = 1;
 		local->buf = *buf;
+                local->preparent = *preparent;
+                local->postparent = *postparent;
 	}
 	cnt = --local->call_count;
 	for (i = local->active + 1; i < child_count; i++) {
@@ -1265,10 +1261,11 @@ ha_symlink_cbk (call_frame_t *frame,
 
 	if (cnt == 0 || i == child_count) {
 		call_stub_t *stub = local->stub;
-		FREE (local->state);
+		GF_FREE (local->state);
 		stub = local->stub;
 		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local->stub->args.symlink.loc.inode, &local->buf);
+			      local->stub->args.symlink.loc.inode, &local->buf,
+                              &local->preparent, &local->postparent);
 		call_stub_destroy (stub);
 		return 0;
 	}
@@ -1311,20 +1308,46 @@ ha_symlink (call_frame_t *frame,
 	ha_private_t *pvt = NULL;
 	int child_count = 0, i = 0;
 	char *stateino = NULL;
+        int32_t op_errno = EINVAL;
 
 	local = frame->local;
 	pvt = this->private;
 	child_count = pvt->child_count;
 
-	frame->local = local = CALLOC (1, sizeof (*local));
+	frame->local = local = GF_CALLOC (1, sizeof (*local), 
+                                          gf_ha_mt_ha_local_t);
+        if (!local) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	local->stub = fop_symlink_stub (frame, ha_symlink, linkname, loc);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	local->op_ret = -1;
 	local->op_errno = ENOTCONN;
-	local->state = CALLOC (1, child_count);
+	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!local->state) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	memcpy (local->state, pvt->state, child_count);
 	local->active = -1;
 
-	stateino = CALLOC (1, child_count);
+	stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!stateino) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
 
 	for (i = 0; i < child_count; i++) {
@@ -1342,6 +1365,12 @@ ha_symlink (call_frame_t *frame,
 		    HA_ACTIVE_CHILD(this, local)->fops->symlink,
 		    linkname, loc);
 	return 0;
+err:
+        local = frame->local;
+        frame->local = NULL;
+        STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        ha_local_wipe (local);
+        return 0;
 }
 
  int32_t
@@ -1350,14 +1379,19 @@ ha_rename_cbk (call_frame_t *frame,
 	       xlator_t *this,
 	       int32_t op_ret,
 	       int32_t op_errno,
-	       struct stat *buf)
+	       struct iatt *buf,
+               struct iatt *preoldparent,
+               struct iatt *postoldparent,
+               struct iatt *prenewparent,
+               struct iatt *postnewparent)
 {
 	int ret = -1;
 
 	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
 
 	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno, buf);
+		STACK_UNWIND (frame, op_ret, op_errno, buf, preoldparent,
+                              postoldparent, prenewparent, postnewparent);
 	}
 	return 0;
 }
@@ -1378,6 +1412,12 @@ ha_rename (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_rename_stub (frame, ha_rename, oldloc, newloc);
+        if (!local->stub) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	STACK_WIND_COOKIE (frame,
 			   ha_rename_cbk,
 			   (void *)(long)local->active,
@@ -1386,7 +1426,7 @@ ha_rename (call_frame_t *frame,
 			   oldloc, newloc);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 	return 0;
 }
 
@@ -1397,8 +1437,9 @@ ha_link_lookup_cbk (call_frame_t *frame,
 		    int32_t op_ret,
 		    int32_t op_errno,
 		    inode_t *inode,
-		    struct stat *buf,
-		    dict_t *dict)
+		    struct iatt *buf,
+                    dict_t *dict,
+                    struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -1434,12 +1475,12 @@ ha_link_lookup_cbk (call_frame_t *frame,
 
 	if (cnt == 0) {
 		call_stub_t *stub = local->stub;
-		FREE (local->state);
+		GF_FREE (local->state);
 		STACK_UNWIND (frame,
 			      local->op_ret,
 			      local->op_errno,
-			      local->stub->args.link.oldloc.inode,
-			      &local->buf);
+			      local->stub->args.link.oldloc.inode, &local->buf,
+                              &local->preparent, &local->postparent);
 		call_stub_destroy (stub);
 	}
 	return 0;
@@ -1452,7 +1493,9 @@ ha_link_cbk (call_frame_t *frame,
 	     int32_t op_ret,
 	     int32_t op_errno,
 	     inode_t *inode,
-	     struct stat *buf)
+             struct iatt *buf,
+             struct iatt *preparent,
+             struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -1485,6 +1528,8 @@ ha_link_cbk (call_frame_t *frame,
 		local->op_ret = 0;
 		local->first_success = 1;
 		local->buf = *buf;
+                local->preparent = *preparent;
+                local->postparent = *postparent;
 	}
 	cnt = --local->call_count;
 	for (i = local->active + 1; i < child_count; i++) {
@@ -1494,9 +1539,11 @@ ha_link_cbk (call_frame_t *frame,
 
 	if (cnt == 0 || i == child_count) {
 		call_stub_t *stub = local->stub;
-		FREE (local->state);
+		GF_FREE (local->state);
 		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, local->stub->args.link.oldloc.inode, &local->buf);
+		STACK_UNWIND (frame, local->op_ret, local->op_errno,
+                              local->stub->args.link.oldloc.inode, &local->buf,
+                              &local->preparent, &local->postparent);
 		call_stub_destroy (stub);
 		return 0;
 	}
@@ -1539,7 +1586,7 @@ ha_link (call_frame_t *frame,
 	ha_private_t *pvt = NULL;
 	int child_count = 0, i = 0;
 	char *stateino = NULL;
-	int32_t ret = 0;
+	int32_t ret = 0, op_errno = 0;
 	uint64_t tmp_stateino = 0;
 
 	ret = inode_ctx_get (newloc->inode, this, &tmp_stateino);
@@ -1551,7 +1598,8 @@ ha_link (call_frame_t *frame,
 	if (stateino == NULL) {
 		gf_log (this->name, GF_LOG_ERROR, 
 			"newloc->inode's ctx is NULL, returning EINVAL");
-		STACK_UNWIND (frame, -1, EINVAL, oldloc->inode, NULL);
+		STACK_UNWIND (frame, -1, EINVAL, oldloc->inode, NULL, NULL,
+                              NULL);
 		return 0;
 	}
 
@@ -1559,11 +1607,30 @@ ha_link (call_frame_t *frame,
 	pvt = this->private;
 	child_count = pvt->child_count;
 
-	frame->local = local = CALLOC (1, sizeof (*local));
+	frame->local = local = GF_CALLOC (1, sizeof (*local), 
+                                          gf_ha_mt_ha_local_t);
+        if (!frame->local) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	local->stub = fop_link_stub (frame, ha_link, oldloc, newloc);
+        if (!local->stub) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	local->op_ret = -1;
 	local->op_errno = ENOTCONN;
-	local->state = CALLOC (1, child_count);
+	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!local->state) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	memcpy (local->state, pvt->state, child_count);
 	local->active = -1;
 
@@ -1582,6 +1649,11 @@ ha_link (call_frame_t *frame,
 		    oldloc,
 		    newloc);
 	return 0;
+err:
+        local = frame->local;
+        frame->local = NULL;
+        STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 int32_t
@@ -1592,7 +1664,9 @@ ha_create_cbk (call_frame_t *frame,
 	       int32_t op_errno,
 	       fd_t *fd,
 	       inode_t *inode,
-	       struct stat *buf)
+	       struct iatt *buf,
+               struct iatt *preparent,
+               struct iatt *postparent)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -1640,6 +1714,8 @@ ha_create_cbk (call_frame_t *frame,
 		if (local->op_ret == -1) {
 			local->op_ret = 0;
 			local->buf = *buf;
+                        local->preparent = *preparent;
+                        local->postparent = *postparent;
 			local->first_success = 1;
 		}
 		local->stub->args.create.flags &= (~O_EXCL);
@@ -1658,8 +1734,9 @@ ha_create_cbk (call_frame_t *frame,
 		call_stub_t *stub = local->stub;
 		STACK_UNWIND (frame, local->op_ret, local->op_errno,
 			      stub->args.create.fd,
-			      stub->args.create.loc.inode, &local->buf);
-		FREE (state);
+			      stub->args.create.loc.inode, &local->buf,
+                              &local->preparent, &local->postparent);
+		GF_FREE (state);
 		call_stub_destroy (stub);
 		return 0;
 	}
@@ -1695,6 +1772,7 @@ ha_create (call_frame_t *frame,
 	char *stateino = NULL;
 	xlator_t **children = NULL;
 	hafd_t *hafdp = NULL;
+        int32_t op_errno = EINVAL;
 
 	local = frame->local;
 	pvt = this->private;
@@ -1702,9 +1780,28 @@ ha_create (call_frame_t *frame,
 	children = pvt->children;
 
 	if (local == NULL) {
-		local = frame->local = CALLOC (1, sizeof (*local));
+                frame->local = local = GF_CALLOC (1, sizeof (*local), 
+                                                  gf_ha_mt_ha_local_t);
+                if (!local) {
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        goto err;
+                }
+
 		local->stub = fop_create_stub (frame, ha_create, loc, flags, mode, fd);
-		local->state = CALLOC (1, child_count);
+                if (!local->stub) {
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        goto err;
+                }
+
+                local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
+                if (!local->state) {
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        goto err;
+                }
+
 		local->active = -1;
 		local->op_ret = -1;
 		local->op_errno = ENOTCONN;
@@ -1718,10 +1815,34 @@ ha_create (call_frame_t *frame,
 			}
 		}
 		/* FIXME handle active -1 */
-		stateino = CALLOC (1, child_count);
-		hafdp = CALLOC (1, sizeof (*hafdp));
-		hafdp->fdstate = CALLOC (1, child_count);
-		hafdp->path = strdup(loc->path);
+		stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
+                if (!stateino) {
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        goto err;
+                }
+
+		hafdp = GF_CALLOC (1, sizeof (*hafdp), gf_ha_mt_hafd_t);
+                if (!hafdp) {
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        goto err;
+                }
+
+		hafdp->fdstate = GF_CALLOC (1, child_count, gf_ha_mt_char);
+                if (!hafdp->fdstate) {
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        goto err;
+                }
+
+		hafdp->path = gf_strdup(loc->path);
+                if (!hafdp->path) {
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        goto err;
+                }
+
 		LOCK_INIT (&hafdp->lock);
 		fd_ctx_set (fd, this, (uint64_t)(long)hafdp);
 		inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
@@ -1733,6 +1854,26 @@ ha_create (call_frame_t *frame,
 		    children[local->active]->fops->create,
 		    loc, flags, mode, fd);
 	return 0;
+err:
+        local = frame->local;
+        frame->local = NULL;
+        STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        ha_local_wipe (local);
+
+        if (stateino) {
+                GF_FREE (stateino);
+                stateino = NULL;
+        }
+
+        if (hafdp) {
+                GF_FREE (hafdp->fdstate);
+
+                GF_FREE (hafdp->path);
+
+                GF_FREE (hafdp);
+        }
+
+        return 0;
 }
 
  int32_t
@@ -1789,7 +1930,7 @@ int32_t
 ha_open (call_frame_t *frame,
 	 xlator_t *this,
 	 loc_t *loc,
-	 int32_t flags, fd_t *fd)
+	 int32_t flags, fd_t *fd, int wbflags)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -1798,6 +1939,7 @@ ha_open (call_frame_t *frame,
 	int cnt = 0, i, child_count = 0, ret = 0;
 	hafd_t *hafdp = NULL;
 	uint64_t tmp_stateino = 0;
+        int32_t op_errno = ENOMEM;
 
 	local = frame->local;
 	pvt = this->private;
@@ -1805,14 +1947,39 @@ ha_open (call_frame_t *frame,
 	child_count = pvt->child_count;
 
 
-	local = frame->local = CALLOC (1, sizeof (*local));
+	frame->local = local = GF_CALLOC (1, sizeof (*local), 
+                                          gf_ha_mt_ha_local_t);
+        if (!local) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+                
 	local->op_ret = -1;
 	local->op_errno = ENOTCONN;
 	local->fd = fd;
 
-	hafdp = CALLOC (1, sizeof (*hafdp));
-	hafdp->fdstate = CALLOC (1, child_count);
-	hafdp->path = strdup (loc->path);
+	hafdp = GF_CALLOC (1, sizeof (*hafdp), gf_ha_mt_hafd_t);
+        if (!hafdp) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
+	hafdp->fdstate = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!hafdp->fdstate) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
+	hafdp->path = gf_strdup (loc->path);
+        if (!hafdp->path) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	hafdp->active = -1;
 	if (pvt->pref_subvol == -1) {
 		hafdp->active = fd->inode->ino % child_count;
@@ -1833,12 +2000,33 @@ ha_open (call_frame_t *frame,
 				    ha_open_cbk,
 				    children[i],
 				    children[i]->fops->open,
-				    loc, flags, fd);
+				    loc, flags, fd, wbflags);
 			if (--cnt == 0)
 				break;
 		}
 	}
 	return 0;
+err:
+        local = frame->local;
+        frame->local = NULL;
+
+	STACK_UNWIND (frame, -1, op_errno, fd);
+        if (hafdp) {
+                if (hafdp->fdstate) {
+                        GF_FREE (hafdp->fdstate);
+                        hafdp->fdstate = NULL;
+                }
+
+                if (hafdp->path) {
+                        GF_FREE (hafdp->path);
+                        hafdp->path = NULL;
+                }
+
+                GF_FREE (hafdp);
+        }
+
+        ha_local_wipe (local);
+        return 0;
 }
 
  int32_t
@@ -1849,7 +2037,8 @@ ha_readv_cbk (call_frame_t *frame,
 	      int32_t op_errno,
 	      struct iovec *vector,
 	      int32_t count,
-	      struct stat *stbuf)
+	      struct iatt *stbuf,
+              struct iobref *iobref)
 {
 	int ret = 0;
 
@@ -1861,7 +2050,8 @@ ha_readv_cbk (call_frame_t *frame,
 			      op_errno,
 			      vector,
 			      count,
-			      stbuf);
+			      stbuf,
+                              iobref);
 	}
 	return 0;
 }
@@ -1883,6 +2073,11 @@ ha_readv (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_readv_stub (frame, ha_readv, fd, size, offset);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_readv_cbk,
@@ -1894,7 +2089,12 @@ ha_readv (call_frame_t *frame,
 			   offset);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
+        local = frame->local;
+        frame->local = NULL;
+
+	STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL, NULL);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -1904,7 +2104,8 @@ ha_writev_cbk (call_frame_t *frame,
 	       xlator_t *this,
 	       int32_t op_ret,
 	       int32_t op_errno,
-	       struct stat *stbuf)
+               struct iatt *prebuf,
+	       struct iatt *postbuf)
 {
 	int ret = 0;
 	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
@@ -1913,7 +2114,8 @@ ha_writev_cbk (call_frame_t *frame,
 		STACK_UNWIND (frame,
 			      op_ret,
 			      op_errno,
-			      stbuf);
+                              prebuf,
+			      postbuf);
 	}
 	return 0;
 }
@@ -1924,7 +2126,8 @@ ha_writev (call_frame_t *frame,
 	   fd_t *fd,
 	   struct iovec *vector,
 	   int32_t count,
-	   off_t off)
+	   off_t off,
+           struct iobref *iobref)
 {
 	ha_local_t *local = NULL;
 	int op_errno = 0;
@@ -1935,7 +2138,13 @@ ha_writev (call_frame_t *frame,
 		goto err;
 	}
 	local = frame->local;
-	local->stub = fop_writev_stub (frame, ha_writev, fd, vector, count, off);
+	local->stub = fop_writev_stub (frame, ha_writev, fd, vector, count, off,
+                                       iobref);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_writev_cbk,
@@ -1945,10 +2154,16 @@ ha_writev (call_frame_t *frame,
 			   fd,
 			   vector,
 			   count,
-			   off);
+			   off,
+                           iobref);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
+        local = frame->local;
+        frame->local = NULL;
+
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+
+        ha_local_wipe (local);
 	return 0;	
 }
 
@@ -1985,6 +2200,12 @@ ha_flush (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_flush_stub (frame, ha_flush, fd);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+                
 	STACK_WIND_COOKIE (frame,
 			   ha_flush_cbk,
 			   (void *)(long)local->active,
@@ -1993,7 +2214,12 @@ ha_flush (call_frame_t *frame,
 			   fd);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2003,7 +2229,9 @@ ha_fsync_cbk (call_frame_t *frame,
 	      void *cookie,
 	      xlator_t *this,
 	      int32_t op_ret,
-	      int32_t op_errno)
+	      int32_t op_errno,
+              struct iatt *prebuf,
+              struct iatt *postbuf)
 {
 	int ret = 0;
 	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
@@ -2032,6 +2260,12 @@ ha_fsync (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_fsync_stub (frame, ha_fsync, fd, flags);
+        if (!local->stub) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
 	STACK_WIND_COOKIE (frame,
 			   ha_fsync_cbk,
 			   (void *)(long)local->active,
@@ -2041,7 +2275,12 @@ ha_fsync (call_frame_t *frame,
 			   flags);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2051,7 +2290,7 @@ ha_fstat_cbk (call_frame_t *frame,
 	      xlator_t *this,
 	      int32_t op_ret,
 	      int32_t op_errno,
-	      struct stat *buf)
+	      struct iatt *buf)
 {
 	int ret = 0;
 
@@ -2082,6 +2321,12 @@ ha_fstat (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_fstat_stub (frame, ha_fstat, fd);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	STACK_WIND_COOKIE (frame,
 			   ha_fstat_cbk,
 			   (void *)(long)local->active,
@@ -2090,7 +2335,12 @@ ha_fstat (call_frame_t *frame,
 			   fd);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno, NULL);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2156,20 +2406,46 @@ ha_opendir (call_frame_t *frame,
 	int cnt = 0, i, child_count = 0, ret = 0;
 	hafd_t *hafdp = NULL;
 	uint64_t tmp_stateino = 0;
+        int32_t op_errno = EINVAL;
 
 	local = frame->local;
 	pvt = this->private;
 	children = pvt->children;
 	child_count = pvt->child_count;
 
-	local = frame->local = CALLOC (1, sizeof (*local));
+	frame->local = local = GF_CALLOC (1, sizeof (*local), 
+                                          gf_ha_mt_ha_local_t);
+        if (!local) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	local->op_ret = -1;
 	local->op_errno = ENOTCONN;
 	local->fd = fd;
 
-	hafdp = CALLOC (1, sizeof (*hafdp));
-	hafdp->fdstate = CALLOC (1, child_count);
-	hafdp->path = strdup (loc->path);
+	hafdp = GF_CALLOC (1, sizeof (*hafdp), gf_ha_mt_hafd_t);
+        if (!hafdp) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
+	hafdp->fdstate = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!hafdp->fdstate) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
+	hafdp->path = gf_strdup (loc->path);
+        if (!hafdp->path) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	LOCK_INIT (&hafdp->lock);
 	fd_ctx_set (fd, this, (uint64_t)(long)hafdp);
 	ret = inode_ctx_get (loc->inode, this, &tmp_stateino);
@@ -2194,6 +2470,26 @@ ha_opendir (call_frame_t *frame,
 		}
 	}
 	return 0;
+err:
+        local = frame->local;
+        frame->local = NULL;
+
+	STACK_UNWIND (frame, -1, op_errno, NULL);
+        ha_local_wipe (local);
+        if (hafdp) {
+                if (hafdp->fdstate) {
+                        GF_FREE (hafdp->fdstate);
+                        hafdp->fdstate = NULL;
+                }
+
+                if (hafdp->path) {
+                        GF_FREE (hafdp->path);
+                        hafdp->path = NULL;
+                }
+
+                GF_FREE (hafdp);
+        }
+        return 0;
 }
 
  int32_t
@@ -2235,7 +2531,14 @@ ha_getdents (call_frame_t *frame,
 		goto err;
 	}
 	local = frame->local;
-	local->stub = fop_getdents_stub (frame, ha_getdents, fd, size, offset, flag);
+	local->stub = fop_getdents_stub (frame, ha_getdents, fd, size, offset,
+                                         flag);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	STACK_WIND_COOKIE (frame,
 			   ha_getdents_cbk,
 			   (void *)(long)local->active,
@@ -2247,7 +2550,12 @@ ha_getdents (call_frame_t *frame,
 			   flag);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno, NULL, 0);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2288,7 +2596,13 @@ ha_setdents (call_frame_t *frame,
 	}
 	local = frame->local;
 
-	local->stub = fop_setdents_stub (frame, ha_setdents, fd, flags, entries, count);
+	local->stub = fop_setdents_stub (frame, ha_setdents, fd, flags, entries,
+                                         count);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_setdents_cbk,
@@ -2301,7 +2615,12 @@ ha_setdents (call_frame_t *frame,
 			   count);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2339,6 +2658,12 @@ ha_fsyncdir (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_fsyncdir_stub (frame, ha_fsyncdir, fd, flags);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	STACK_WIND_COOKIE (frame,
 			   ha_fsyncdir_cbk,
 			   (void *)(long)local->active,
@@ -2348,7 +2673,12 @@ ha_fsyncdir (call_frame_t *frame,
 			   flags);
 	return 0;
 err:
-	STACK_UNWIND (frame, -1, op_errno);
+        local = frame->local;
+        frame->local = NULL;
+ 
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2361,15 +2691,27 @@ ha_statfs_cbk (call_frame_t *frame,
 	       int32_t op_errno,
 	       struct statvfs *buf)
 {
-	int ret = -1;
+        ha_local_t   *local = NULL;
+        ha_private_t *priv  = NULL;
 
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
-	}
+        local = frame->local;
+        if (-1 == op_ret) {
+                local->active = (local->active + 1) % priv->child_count;
+                local->tries--;
+                if (!local->tries)
+                        goto out;
+                
+                STACK_WIND (frame, ha_statfs_cbk,
+                            HA_ACTIVE_CHILD(this, local),
+                            HA_ACTIVE_CHILD(this, local)->fops->statfs, 
+                            &local->loc);
+                return 0;
+        }
+
+ out:
+        loc_wipe (&local->loc);
+        STACK_UNWIND (frame, op_ret, op_errno, buf);
+        
 	return 0;
 }
 
@@ -2378,23 +2720,29 @@ ha_statfs (call_frame_t *frame,
 	   xlator_t *this,
 	   loc_t *loc)
 {
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-
-	local->stub = fop_statfs_stub (frame, ha_statfs, loc);
-	STACK_WIND_COOKIE (frame,
-			   ha_statfs_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->statfs,
-			   loc);
+        ha_private_t *priv = NULL;
+	ha_local_t   *local = NULL;
+	int           op_errno = 0;
+
+        /* The normal way of handling failover doesn't work here
+         * as loc->inode may be null in this case.
+         */
+        local = GF_CALLOC (1, sizeof (*local), 
+                           gf_ha_mt_ha_local_t);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        priv = this->private;
+        frame->local  = local;
+        local->active = priv->pref_subvol;
+        if (-1 == local->active)
+                local->active = 0;
+        local->tries  = priv->child_count;
+        loc_copy (&local->loc, loc);
+
+	STACK_WIND (frame, ha_statfs_cbk, HA_ACTIVE_CHILD(this, local),
+                    HA_ACTIVE_CHILD(this, local)->fops->statfs, loc);
 	return 0;
 err:
 	STACK_UNWIND (frame, -1, op_errno, NULL);
@@ -2437,6 +2785,12 @@ ha_setxattr (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_setxattr_stub (frame, ha_setxattr, loc, dict, flags);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	STACK_WIND_COOKIE (frame,
 			   ha_setxattr_cbk,
 			   (void *)(long)local->active,
@@ -2447,7 +2801,12 @@ ha_setxattr (call_frame_t *frame,
 			   flags);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2488,6 +2847,12 @@ ha_getxattr (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_getxattr_stub (frame, ha_getxattr, loc, name);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	STACK_WIND_COOKIE (frame,
 			   ha_getxattr_cbk,
 			   (void *)(long)local->active,
@@ -2497,7 +2862,12 @@ ha_getxattr (call_frame_t *frame,
 			   name);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno, NULL);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2536,6 +2906,11 @@ ha_xattrop (call_frame_t *frame,
 	local = frame->local;
 
 	local->stub = fop_xattrop_stub (frame, ha_xattrop, loc, flags, dict);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_xattrop_cbk,
@@ -2547,7 +2922,12 @@ ha_xattrop (call_frame_t *frame,
 			   dict);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno, dict);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2583,6 +2963,11 @@ ha_fxattrop (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_fxattrop_stub (frame, ha_fxattrop, fd, flags, dict);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_fxattrop_cbk,
@@ -2594,7 +2979,12 @@ ha_fxattrop (call_frame_t *frame,
 			   dict);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno, dict);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2632,6 +3022,11 @@ ha_removexattr (call_frame_t *frame,
 	local = frame->local;
 	
 	local->stub = fop_removexattr_stub (frame, ha_removexattr, loc, name);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_removexattr_cbk,
@@ -2642,7 +3037,12 @@ ha_removexattr (call_frame_t *frame,
 			   name);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
@@ -2652,7 +3052,7 @@ ha_lk_setlk_unlck_cbk (call_frame_t *frame,
 		       xlator_t *this,
 		       int32_t op_ret,
 		       int32_t op_errno,
-		       struct flock *lock)
+		       struct gf_flock *lock)
 {
 	ha_local_t *local = NULL;
 	int cnt = 0;
@@ -2668,7 +3068,7 @@ ha_lk_setlk_unlck_cbk (call_frame_t *frame,
 
 	if (cnt == 0) {
 		stub = local->stub;
-		FREE (local->state);
+		GF_FREE (local->state);
 		if (stub->args.lk.lock.l_type == F_UNLCK) {
 			STACK_UNWIND (frame, local->op_ret, local->op_errno, &stub->args.lk.lock);
 		} else {
@@ -2685,7 +3085,7 @@ ha_lk_setlk_cbk (call_frame_t *frame,
 		 xlator_t *this,
 		 int32_t op_ret,
 		 int32_t op_errno,
-		 struct flock *lock)
+		 struct gf_flock *lock)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -2717,7 +3117,7 @@ ha_lk_setlk_cbk (call_frame_t *frame,
 		}
 		if (i == child_count) {
 			call_stub_t *stub = local->stub;
-			FREE (local->state);
+			GF_FREE (local->state);
 			STACK_UNWIND (frame, 0, op_errno, &stub->args.lk.lock);
 			call_stub_destroy (stub);
 			return 0;
@@ -2741,7 +3141,7 @@ ha_lk_setlk_cbk (call_frame_t *frame,
 				cnt++;
 		}
 		if (cnt) {
-			struct flock lock;
+			struct gf_flock lock;
 			lock = local->stub->args.lk.lock;
 			for (i = 0; i < child_count; i++) {
 				if (state[i]) {
@@ -2758,7 +3158,7 @@ ha_lk_setlk_cbk (call_frame_t *frame,
 			}
 			return 0;
 		} else {
-			FREE (local->state);
+			GF_FREE (local->state);
 			call_stub_destroy (local->stub);
 			STACK_UNWIND (frame,
 				      op_ret,
@@ -2775,7 +3175,7 @@ ha_lk_getlk_cbk (call_frame_t *frame,
 		 xlator_t *this,
 		 int32_t op_ret,
 		 int32_t op_errno,
-		 struct flock *lock)
+		 struct gf_flock *lock)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -2792,7 +3192,7 @@ ha_lk_getlk_cbk (call_frame_t *frame,
 	prev_frame = cookie;
 
 	if (op_ret == 0) {
-		FREE (local->state);
+		GF_FREE (local->state);
 		call_stub_destroy (local->stub);
 		STACK_UNWIND (frame, 0, 0, lock);
 		return 0;
@@ -2809,7 +3209,7 @@ ha_lk_getlk_cbk (call_frame_t *frame,
 	}
 
 	if (i == child_count) {
-		FREE (local->state);
+		GF_FREE (local->state);
 		call_stub_destroy (local->stub);
 		STACK_UNWIND (frame, op_ret, op_errno, lock);
 		return 0;
@@ -2830,7 +3230,7 @@ ha_lk (call_frame_t *frame,
        xlator_t *this,
        fd_t *fd,
        int32_t cmd,
-       struct flock *lock)
+       struct gf_flock *lock)
 {
 	ha_local_t *local = NULL;
 	ha_private_t *pvt = NULL;
@@ -2839,6 +3239,7 @@ ha_lk (call_frame_t *frame,
 	int child_count = 0, i = 0, cnt = 0, ret = 0;
 	xlator_t **children = NULL;
 	uint64_t tmp_hafdp = 0;
+        int32_t op_errno = EINVAL;
 
 	local = frame->local;
 	pvt = this->private;
@@ -2849,7 +3250,14 @@ ha_lk (call_frame_t *frame,
 		gf_log (this->name, GF_LOG_ERROR, "fd_ctx_get failed");
 
 	if (local == NULL) {
-		local = frame->local = CALLOC (1, sizeof (*local));
+                local = frame->local = GF_CALLOC (1, sizeof (*local),
+                                                  gf_ha_mt_ha_local_t);
+                if (!local) {
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        goto err;
+                }
+                        
 		local->active = -1;
 		local->op_ret = -1;
 		local->op_errno = ENOTCONN;
@@ -2857,12 +3265,24 @@ ha_lk (call_frame_t *frame,
 	hafdp = (hafd_t *)(long)tmp_hafdp;
 
 	if (local->active == -1) {
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
+                op_errno = ENOTCONN;
+                goto err;
 	}
 
 	local->stub = fop_lk_stub (frame, ha_lk, fd, cmd, lock);
-	local->state = CALLOC (1, child_count);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
+	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
+        if (!local->state) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
 	state = hafdp->fdstate;
 	LOCK (&hafdp->lock);
 	memcpy (local->state, state, child_count);
@@ -2910,6 +3330,14 @@ ha_lk (call_frame_t *frame,
 			    lock);
 	}
 	return 0;
+err:
+        local = frame->local;
+        frame->local = NULL;
+
+        STACK_UNWIND (frame, -1, op_errno, NULL);
+
+        ha_local_wipe (local);
+        return 0;
 }
 
  int32_t
@@ -2933,9 +3361,10 @@ ha_inode_entry_lk_cbk (call_frame_t *frame,
 int32_t
 ha_inodelk (call_frame_t *frame,
 	    xlator_t *this,
+            const char *volume,
 	    loc_t *loc,
 	    int32_t cmd,
-	    struct flock *lock)
+	    struct gf_flock *lock)
 {
 	ha_local_t *local = NULL;
 	int op_errno = 0;
@@ -2946,13 +3375,15 @@ ha_inodelk (call_frame_t *frame,
 		goto err;
 	}
 	local = frame->local;
-	local->stub = fop_inodelk_stub (frame, ha_inodelk, loc, cmd, lock);
+	local->stub = fop_inodelk_stub (frame, ha_inodelk, volume,
+                                        loc, cmd, lock);
 	STACK_WIND_COOKIE (frame,
 			   ha_inode_entry_lk_cbk,
 			   (void *)(long)local->active,
 			   HA_ACTIVE_CHILD(this, local),
 			   HA_ACTIVE_CHILD(this, local)->fops->inodelk,
-			   loc,
+			   volume,
+                           loc,
 			   cmd,
 			   lock);
 	return 0;
@@ -2964,6 +3395,7 @@ err:
 int32_t
 ha_entrylk (call_frame_t *frame,
 	    xlator_t *this,
+            const char *volume,
 	    loc_t *loc,
 	    const char *basename,
 	    entrylk_cmd cmd,
@@ -2978,13 +3410,14 @@ ha_entrylk (call_frame_t *frame,
 		goto err;
 	}
 	local = frame->local;
-	local->stub = fop_entrylk_stub (frame, ha_entrylk, loc, basename, cmd, type);
+	local->stub = fop_entrylk_stub (frame, ha_entrylk, volume,
+                                        loc, basename, cmd, type);
 	STACK_WIND_COOKIE (frame,
 			   ha_inode_entry_lk_cbk,
 			   (void *)(long)local->active,
 			   HA_ACTIVE_CHILD(this, local),
 			   HA_ACTIVE_CHILD(this, local)->fops->entrylk,
-			   loc, basename, cmd, type);
+			   volume, loc, basename, cmd, type);
 	return 0;
 err:
 	STACK_UNWIND (frame, -1, op_errno);
@@ -3029,6 +3462,11 @@ ha_checksum (call_frame_t *frame,
 	}
 	local = frame->local;
 	local->stub = fop_checksum_stub (frame, ha_checksum, loc, flag);
+        if (!local->stub) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
 	STACK_WIND_COOKIE (frame,
 			   ha_checksum_cbk,
@@ -3039,17 +3477,18 @@ ha_checksum (call_frame_t *frame,
 			   flag);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
 int32_t
-ha_readdir_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		gf_dirent_t *entries)
+ha_common_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
 {
 	int ret = 0;
 
@@ -3060,11 +3499,15 @@ ha_readdir_cbk (call_frame_t *frame,
 }
 
 int32_t
-ha_readdir (call_frame_t *frame,
-	    xlator_t *this,
-	    fd_t *fd,
-	    size_t size,
-	    off_t off)
+ha_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off);
+
+int32_t
+ha_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off);
+int32_t
+ha_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t off, int whichop)
 {
 	ha_local_t *local = NULL;
 	int op_errno = 0;
@@ -3075,19 +3518,58 @@ ha_readdir (call_frame_t *frame,
 		goto err;
 	}
 	local = frame->local;
-	local->stub = fop_readdir_stub (frame, ha_readdir, fd, size, off);
-	STACK_WIND_COOKIE (frame,
-			   ha_readdir_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->readdir,
-			   fd, size, off);
+        if (whichop == GF_FOP_READDIR)
+                local->stub = fop_readdir_stub (frame, ha_readdir, fd, size,
+                                                off);
+        else
+                local->stub = fop_readdirp_stub (frame, ha_readdirp, fd, size,
+                                                 off);
+        if (!local->stub) {
+                op_errno = ENOMEM; 
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
+
+        if (whichop == GF_FOP_READDIR)
+                STACK_WIND_COOKIE (frame, ha_common_readdir_cbk,
+                                   (void *)(long)local->active,
+                                   HA_ACTIVE_CHILD(this, local),
+                                   HA_ACTIVE_CHILD(this, local)->fops->readdir,
+                                   fd, size, off);
+        else
+                STACK_WIND_COOKIE (frame, ha_common_readdir_cbk,
+                                   (void *)(long)local->active,
+                                   HA_ACTIVE_CHILD(this, local),
+                                   HA_ACTIVE_CHILD(this, local)->fops->readdirp,
+                                   fd, size, off);
 	return 0;
 err:
+        local = frame->local;
+        frame->local = NULL;
+
 	STACK_UNWIND (frame, -1, ENOTCONN, NULL);
+
+        ha_local_wipe (local);
 	return 0;
 }
 
+
+int32_t
+ha_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off)
+{
+        ha_do_readdir (frame, this, fd, size, off, GF_FOP_READDIR);
+        return 0;
+}
+
+int32_t
+ha_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off)
+{
+        ha_do_readdir (frame, this, fd, size, off, GF_FOP_READDIRP);
+        return 0;
+}
+
 /* Management operations */
 
  int32_t
@@ -3148,8 +3630,16 @@ ha_stats (call_frame_t *frame,
 	ha_private_t *pvt = NULL;
 	xlator_t **children = NULL;
 	int i = 0;
+        int32_t op_errno = EINVAL;
+
+	local = frame->local = GF_CALLOC (1, sizeof (*local),
+                                          gf_ha_mt_ha_local_t);
+        if (!local) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
-	local = frame->local = CALLOC (1, sizeof (*local));
 	pvt = this->private;
 	children = pvt->children;
 	for (i = 0; i < pvt->child_count; i++) {
@@ -3158,8 +3648,8 @@ ha_stats (call_frame_t *frame,
 	}
 
 	if (i == pvt->child_count) {
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
+                op_errno = ENOTCONN;
+                goto err;
 	}
 	local->flags = flags;
 
@@ -3169,6 +3659,16 @@ ha_stats (call_frame_t *frame,
 		    children[i]->mops->stats,
 		    flags);
 	return 0;
+
+err:
+        local = frame->local;
+        frame->local = NULL;
+
+        STACK_UNWIND (frame, -1, ENOTCONN, NULL);
+
+        ha_local_wipe (local);
+        return 0;
+
 }
 
 
@@ -3232,20 +3732,27 @@ ha_getspec (call_frame_t *frame,
 	ha_private_t *pvt = NULL;
 	xlator_t **children = NULL;
 	int i = 0;
+        int32_t op_errno = EINVAL;
+
+	local = frame->local = GF_CALLOC (1, sizeof (*local), 
+                                          gf_ha_mt_ha_local_t);
+        if (!local) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto err;
+        }
 
-	local = frame->local = CALLOC (1, sizeof (*local));
 	pvt = this->private;
 	children = pvt->children;
 
-	local = frame->local = CALLOC (1, sizeof (*local));
 	for (i = 0; i < pvt->child_count; i++) {
 		if (pvt->state[i])
 			break;
 	}
 
 	if (i == pvt->child_count) {
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
+                op_errno = ENOTCONN;
+                goto err;
 	}
 	local->flags = flags;
 	local->pattern = (char *)key;
@@ -3256,6 +3763,15 @@ ha_getspec (call_frame_t *frame,
 		    children[i]->mops->getspec,
 		    key, flags);
 	return 0;
+err:
+        local = frame->local;
+        frame->local = NULL;
+
+        STACK_UNWIND (frame, -1, ENOTCONN, NULL);
+
+        ha_local_wipe (local);
+        return 0;
+
 }
 
 int32_t
@@ -3273,8 +3789,8 @@ ha_closedir (xlator_t *this,
 	}
 	hafdp = (hafd_t *)(long)tmp_hafdp;
 
-	FREE (hafdp->fdstate);
-	FREE (hafdp->path);
+	GF_FREE (hafdp->fdstate);
+	GF_FREE (hafdp->path);
 	LOCK_DESTROY (&hafdp->lock);
 	return 0;
 }
@@ -3294,8 +3810,8 @@ ha_close (xlator_t *this,
 	}
 	hafdp = (hafd_t *)(long)tmp_hafdp;
 
-	FREE (hafdp->fdstate);
-	FREE (hafdp->path);
+	GF_FREE (hafdp->fdstate);
+	GF_FREE (hafdp->path);
 	LOCK_DESTROY (&hafdp->lock);
 	return 0;
 }
@@ -3366,6 +3882,25 @@ notify (xlator_t *this,
 	return 0;
 }
 
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_ha_mt_end + 1);
+        
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                                "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
 int
 init (xlator_t *this)
 {
@@ -3373,6 +3908,7 @@ init (xlator_t *this)
 	xlator_list_t *trav = NULL;
 	int count = 0, ret = 0;
 
+
 	if (!this->children) {
 		gf_log (this->name,GF_LOG_ERROR, 
 			"FATAL: ha should have one or more child defined");
@@ -3385,7 +3921,7 @@ init (xlator_t *this)
 	}
   
 	trav = this->children;
-	pvt = CALLOC (1, sizeof (ha_private_t));
+	pvt = GF_CALLOC (1, sizeof (ha_private_t), gf_ha_mt_ha_private_t);
 
 	ret = dict_get_int32 (this->options, "preferred-subvolume", 
 			      &pvt->pref_subvol);
@@ -3400,7 +3936,8 @@ init (xlator_t *this)
 	}
 
 	pvt->child_count = count;
-	pvt->children = CALLOC (count, sizeof (xlator_t*));
+	pvt->children = GF_CALLOC (count, sizeof (xlator_t*), 
+                                   gf_ha_mt_xlator_t);
 
 	trav = this->children;
 	count = 0;
@@ -3410,7 +3947,7 @@ init (xlator_t *this)
 		trav = trav->next;
 	}
 
-	pvt->state = CALLOC (1, count);
+	pvt->state = GF_CALLOC (1, count, gf_ha_mt_char);
 	this->private = pvt;
 	return 0;
 }
@@ -3420,7 +3957,7 @@ fini (xlator_t *this)
 {
 	ha_private_t *priv = NULL;
 	priv = this->private;
-	FREE (priv);
+	GF_FREE (priv);
 	return;
 }
 
@@ -3436,10 +3973,7 @@ struct xlator_fops fops = {
 	.symlink     = ha_symlink,
 	.rename      = ha_rename,
 	.link        = ha_link,
-	.chmod       = ha_chmod,
-	.chown       = ha_chown,
 	.truncate    = ha_truncate,
-	.utimens     = ha_utimens,
 	.create      = ha_create,
 	.open        = ha_open,
 	.readv       = ha_readv,
@@ -3452,24 +3986,20 @@ struct xlator_fops fops = {
 	.removexattr = ha_removexattr,
 	.opendir     = ha_opendir,
 	.readdir     = ha_readdir,
+	.readdirp    = ha_readdirp,
 	.getdents    = ha_getdents,
 	.fsyncdir    = ha_fsyncdir,
 	.access      = ha_access,
 	.ftruncate   = ha_ftruncate,
 	.fstat       = ha_fstat,
 	.lk          = ha_lk,
-	.fchmod      = ha_fchmod,
-	.fchown      = ha_fchown,
 	.setdents    = ha_setdents,
 	.lookup_cbk  = ha_lookup_cbk,
 	.checksum    = ha_checksum,
 	.xattrop     = ha_xattrop,
-	.fxattrop    = ha_fxattrop
-};
-
-struct xlator_mops mops = {
-	.stats   = ha_stats,
-	.getspec = ha_getspec,
+	.fxattrop    = ha_fxattrop,
+        .setattr     = ha_setattr,
+        .fsetattr    = ha_fsetattr,
 };
 
 struct xlator_cbks cbks = {
diff --git a/xlators/cluster/ha/src/ha.h b/xlators/cluster/ha/src/ha.h
index 77a04f165..e2ed7eaa6 100644
--- a/xlators/cluster/ha/src/ha.h
+++ b/xlators/cluster/ha/src/ha.h
@@ -1,25 +1,17 @@
 /*
-  Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __HA_H_
 #define __HA_H_
 
+#include "ha-mem-types.h"
+
 typedef struct {
 	call_stub_t *stub;
 	int32_t op_ret, op_errno;
@@ -27,8 +19,10 @@ typedef struct {
 	int32_t call_count;
 	char *state, *pattern;
 	dict_t *dict;
-	loc_t *loc;
-	struct stat buf;
+	loc_t loc;
+	struct iatt buf;
+        struct iatt postparent;
+        struct iatt preparent;
 	fd_t *fd;
 	inode_t *inode;
 	int32_t flags;
diff --git a/xlators/cluster/map/src/Makefile.am b/xlators/cluster/map/src/Makefile.am
index 44ee4d9ee..a278b05e2 100644
--- a/xlators/cluster/map/src/Makefile.am
+++ b/xlators/cluster/map/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = map.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-map_la_LDFLAGS = -module -avoidversion 
+map_la_LDFLAGS = -module -avoid-version 
 
 map_la_SOURCES = map.c map-helper.c
 map_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = map.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/map/src/map-helper.c b/xlators/cluster/map/src/map-helper.c
index 4e51219d4..851397b68 100644
--- a/xlators/cluster/map/src/map-helper.c
+++ b/xlators/cluster/map/src/map-helper.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2009-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -256,14 +246,15 @@ verify_dir_and_assign_subvol (xlator_t *this,
 				goto out;
 			}
 
-			tmp_map = CALLOC (1, sizeof (struct map_pattern));
+			tmp_map = GF_CALLOC (1, sizeof (struct map_pattern),
+                                             gf_map_mt_map_pattern);
 			tmp_map->xl = trav->xlator;
 			tmp_map->dir_len = strlen (directory);
 
 			/* make sure that the top level directory starts 
 			 * with '/' and ends without '/'
 			 */
-			tmp_map->directory = strdup (directory);
+			tmp_map->directory = gf_strdup (directory);
 			if (directory[tmp_map->dir_len - 1] == '/') {
 				tmp_map->dir_len--;
 			}
diff --git a/xlators/cluster/map/src/map-mem-types.h b/xlators/cluster/map/src/map-mem-types.h
new file mode 100644
index 000000000..3e89f4736
--- /dev/null
+++ b/xlators/cluster/map/src/map-mem-types.h
@@ -0,0 +1,24 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __MAP_MEM_TYPES_H__
+#define __MAP_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_map_mem_types_ {
+        gf_map_mt_map_private_t = gf_common_mt_end + 1,
+        gf_map_mt_map_local_t,
+        gf_map_mt_map_xlator_array,
+        gf_map_mt_map_pattern,
+        gf_map_mt_end
+};
+#endif
+
diff --git a/xlators/cluster/map/src/map.c b/xlators/cluster/map/src/map.c
index 8c4b7c83c..6150a33ce 100644
--- a/xlators/cluster/map/src/map.c
+++ b/xlators/cluster/map/src/map.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -25,104 +15,910 @@
 #include "xlator.h"
 #include "map.h"
 
-/* For <op>_cbk functions */
-#include "defaults.c"
+/* TODO : 
+ *     -> support for 'get' 'put' API in through xattrs.
+ *     -> define the behavior of notify()
+ */
+
+static int32_t
+map_stat_cbk (call_frame_t *frame,
+		  void *cookie,
+		  xlator_t *this,
+		  int32_t op_ret,
+		  int32_t op_errno,
+		  struct iatt *buf)
+
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, buf);
+	return 0;
+}
+
+static int32_t
+map_setattr_cbk (call_frame_t *frame,
+                 void *cookie,
+                 xlator_t *this,
+                 int32_t op_ret,
+                 int32_t op_errno,
+                 struct iatt *statpre,
+                 struct iatt *statpost)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+
+	map_itransform (this, prev->this, statpre->ia_ino, &statpre->ia_ino);
+	map_itransform (this, prev->this, statpost->ia_ino, &statpost->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, statpre, statpost);
+	return 0;
+}
+
+static int32_t
+map_fsetattr_cbk (call_frame_t *frame,
+                  void *cookie,
+                  xlator_t *this,
+                  int32_t op_ret,
+                  int32_t op_errno,
+                  struct iatt *statpre,
+                  struct iatt *statpost)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+
+	map_itransform (this, prev->this, statpre->ia_ino, &statpre->ia_ino);
+	map_itransform (this, prev->this, statpost->ia_ino, &statpost->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, statpre, statpost);
+	return 0;
+}
+
+static int32_t
+map_truncate_cbk (call_frame_t *frame,
+		      void *cookie,
+		      xlator_t *this,
+		      int32_t op_ret,
+		      int32_t op_errno,
+		      struct iatt *prebuf,
+                      struct iatt *postbuf)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, postbuf->ia_ino, &postbuf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
+	return 0;
+}
+
+static int32_t
+map_ftruncate_cbk (call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iatt *prebuf,
+                       struct iatt *postbuf)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, postbuf->ia_ino, &postbuf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
+	return 0;
+}
+
+
+static int32_t
+map_access_cbk (call_frame_t *frame,
+		    void *cookie,
+		    xlator_t *this,
+		    int32_t op_ret,
+		    int32_t op_errno)
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+static int32_t
+map_readlink_cbk (call_frame_t *frame,
+		      void *cookie,
+		      xlator_t *this,
+		      int32_t op_ret,
+		      int32_t op_errno,
+		      const char *path,
+                      struct iatt *sbuf)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
+	return 0;
+}
+
+static int32_t
+map_unlink_cbk (call_frame_t *frame,
+		    void *cookie,
+		    xlator_t *this,
+		    int32_t op_ret,
+		    int32_t op_errno,
+                    struct iatt *preparent,
+                    struct iatt *postparent)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
+	return 0;
+}
+
+static int32_t
+map_rmdir_cbk (call_frame_t *frame,
+		   void *cookie,
+		   xlator_t *this,
+		   int32_t op_ret,
+		   int32_t op_errno,
+                   struct iatt *preparent,
+                   struct iatt *postparent)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
+	return 0;
+}
+
+
+static int32_t
+map_rename_cbk (call_frame_t *frame,
+		    void *cookie,
+		    xlator_t *this,
+		    int32_t op_ret,
+		    int32_t op_errno,
+		    struct iatt *buf,
+                    struct iatt *preoldparent,
+                    struct iatt *postoldparent,
+                    struct iatt *prenewparent,
+                    struct iatt *postnewparent)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, buf);
+	return 0;
+}
+
+static int32_t
+map_link_cbk (call_frame_t *frame,
+		  void *cookie,
+		  xlator_t *this,
+		  int32_t op_ret,
+		  int32_t op_errno,
+		  inode_t *inode,
+                  struct iatt *buf,
+                  struct iatt *preparent,
+                  struct iatt *postparent)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, inode,	buf);
+	return 0;
+}
+
+static int32_t
+map_open_cbk (call_frame_t *frame,
+		  void *cookie,
+		  xlator_t *this,
+		  int32_t op_ret,
+		  int32_t op_errno,
+		  fd_t *fd)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, fd);
+	return 0;
+}
+
+static int32_t
+map_readv_cbk (call_frame_t *frame,
+               void *cookie,
+               xlator_t *this,
+               int32_t op_ret,
+               int32_t op_errno,
+               struct iovec *vector,
+               int32_t count,
+               struct iatt *stbuf,
+               struct iobref *iobref)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, stbuf->ia_ino, &stbuf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf, iobref);
+	return 0;
+}
+
+static int32_t
+map_writev_cbk (call_frame_t *frame,
+		    void *cookie,
+		    xlator_t *this,
+		    int32_t op_ret,
+		    int32_t op_errno,
+                    struct iatt *prebuf,
+		    struct iatt *postbuf)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, postbuf->ia_ino, &postbuf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
+	return 0;
+}
+
+static int32_t
+map_flush_cbk (call_frame_t *frame,
+		   void *cookie,
+		   xlator_t *this,
+		   int32_t op_ret,
+		   int32_t op_errno)
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+
+static int32_t
+map_fsync_cbk (call_frame_t *frame,
+		   void *cookie,
+		   xlator_t *this,
+		   int32_t op_ret,
+		   int32_t op_errno,
+                   struct iatt *prebuf,
+                   struct iatt *postbuf)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
+	return 0;
+}
+
+
+static int32_t
+map_fstat_cbk (call_frame_t *frame,
+		   void *cookie,
+		   xlator_t *this,
+		   int32_t op_ret,
+		   int32_t op_errno,
+		   struct iatt *buf)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, buf);
+	return 0;
+}
+
+
+static int32_t
+map_getdents_cbk (call_frame_t *frame,
+		      void *cookie,
+		      xlator_t *this,
+		      int32_t op_ret,
+		      int32_t op_errno,
+		      dir_entry_t *entries,
+		      int32_t count)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, entries, count);
+	return 0;
+}
 
 
+static int32_t
+map_setdents_cbk (call_frame_t *frame,
+		      void *cookie,
+		      xlator_t *this,
+		      int32_t op_ret,
+		      int32_t op_errno)
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+static int32_t
+map_fsyncdir_cbk (call_frame_t *frame,
+		      void *cookie,
+		      xlator_t *this,
+		      int32_t op_ret,
+		      int32_t op_errno)
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+
+static int32_t
+map_setxattr_cbk (call_frame_t *frame,
+		      void *cookie,
+		      xlator_t *this,
+		      int32_t op_ret,
+		      int32_t op_errno)
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+
+static int32_t
+map_fsetxattr_cbk (call_frame_t *frame,
+                       void *cookie,
+                       xlator_t *this,
+                       int32_t op_ret,
+                       int32_t op_errno)
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+
+static int32_t
+map_fgetxattr_cbk (call_frame_t *frame,
+                       void *cookie,
+                       xlator_t *this,
+                       int32_t op_ret,
+                       int32_t op_errno,
+                       dict_t *dict)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, dict);
+	return 0;
+}
+
+
+
+static int32_t
+map_getxattr_cbk (call_frame_t *frame,
+                      void *cookie,
+                      xlator_t *this,
+                      int32_t op_ret,
+                      int32_t op_errno,
+                      dict_t *dict)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, dict);
+	return 0;
+}
+
 int32_t
-map_stat (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc)
+map_xattrop_cbk (call_frame_t *frame,
+		     void *cookie,
+		     xlator_t *this,
+		     int32_t op_ret,
+		     int32_t op_errno,
+		     dict_t *dict)
 {
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
+	STACK_UNWIND (frame, op_ret, op_errno, dict);
+	return 0;
+}
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+int32_t
+map_fxattrop_cbk (call_frame_t *frame,
+		      void *cookie,
+		      xlator_t *this,
+		      int32_t op_ret,
+		      int32_t op_errno,
+		      dict_t *dict)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, dict);
+	return 0;
+}
 
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
+static int32_t
+map_removexattr_cbk (call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno)
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+
+static int32_t
+map_lk_cbk (call_frame_t *frame,
+		void *cookie,
+		xlator_t *this,
+		int32_t op_ret,
+		int32_t op_errno,
+		struct gf_flock *lock)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, lock);
+	return 0;
+}
+
+
+static int32_t
+map_inodelk_cbk (call_frame_t *frame, void *cookie,
+		     xlator_t *this, int32_t op_ret, int32_t op_errno)
+
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+
+
+static int32_t
+map_finodelk_cbk (call_frame_t *frame, void *cookie,
+		      xlator_t *this, int32_t op_ret, int32_t op_errno)
+
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+
+static int32_t
+map_entrylk_cbk (call_frame_t *frame, void *cookie,
+		     xlator_t *this, int32_t op_ret, int32_t op_errno)
+
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+
+static int32_t
+map_fentrylk_cbk (call_frame_t *frame, void *cookie,
+		      xlator_t *this, int32_t op_ret, int32_t op_errno)
+
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+static int32_t
+map_newentry_cbk (call_frame_t *frame,
+		  void *cookie,
+		  xlator_t *this,
+		  int32_t op_ret,
+		  int32_t op_errno,
+		  inode_t *inode,
+                  struct iatt *buf,
+                  struct iatt *preparent,
+                  struct iatt *postparent)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
+	return 0;
+
+}
+
+
+static int32_t
+map_create_cbk (call_frame_t *frame,
+		void *cookie,
+		xlator_t *this,
+		int32_t op_ret,
+		int32_t op_errno,
+		fd_t *fd,
+		inode_t *inode,
+		struct iatt *buf,
+                struct iatt *preparent,
+                struct iatt *postparent)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
+	return 0;
+}
+
+
+/*
+ * map_normalize_stats -
+ */
+void
+map_normalize_stats (struct statvfs *buf,
+		     unsigned long bsize,
+		     unsigned long frsize)
+{
+	double factor;
+
+	if (buf->f_bsize != bsize) {
+		factor = ((double) buf->f_bsize) / bsize;
+		buf->f_bsize  = bsize;
+		buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
+		buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
+	}
+  
+	if (buf->f_frsize != frsize) {
+		factor = ((double) buf->f_frsize) / frsize;
+		buf->f_frsize = frsize;
+		buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
+	}
+}
+
+
+int32_t
+map_statfs_cbk (call_frame_t *frame,
+		void *cookie,
+		xlator_t *this,
+		int32_t op_ret,
+		int32_t op_errno,
+		struct statvfs *stbuf)
+{
+	struct statvfs *dict_buf = NULL;
+	map_local_t  *local = NULL;
+	int           this_call_cnt = 0;
+	unsigned long bsize;
+	unsigned long frsize;
+
+	local = frame->local;
+
+	LOCK (&frame->lock);
+	{
+		this_call_cnt = --local->call_count;
+
+		if (op_ret == -1) {
+			local->op_errno = op_errno;
+			goto unlock;
+		}
+		local->op_ret = 0;
+		
+		/* when a call is successfull, add it to local->dict */
+		dict_buf = &local->statvfs;
+		
+		if (dict_buf->f_bsize != 0) {
+			bsize  = max (dict_buf->f_bsize, 
+				      stbuf->f_bsize);
+			
+			frsize = max (dict_buf->f_frsize, 
+				      stbuf->f_frsize);
+			map_normalize_stats(dict_buf, bsize, frsize);
+			map_normalize_stats(stbuf, bsize, frsize);
+		} else {
+			dict_buf->f_bsize   = stbuf->f_bsize;
+			dict_buf->f_frsize  = stbuf->f_frsize;
+		}
+		
+		dict_buf->f_blocks += stbuf->f_blocks;
+		dict_buf->f_bfree  += stbuf->f_bfree;
+		dict_buf->f_bavail += stbuf->f_bavail;
+		dict_buf->f_files  += stbuf->f_files;
+		dict_buf->f_ffree  += stbuf->f_ffree;
+		dict_buf->f_favail += stbuf->f_favail;
+		dict_buf->f_fsid    = stbuf->f_fsid;
+		dict_buf->f_flag    = stbuf->f_flag;
+		dict_buf->f_namemax = stbuf->f_namemax;
 	}
+unlock:
+	UNLOCK (&frame->lock);
 
-	STACK_WIND (frame,
-		    default_stat_cbk,
-		    subvol,
-		    subvol->fops->stat,
-		    loc);
+	if (!this_call_cnt) {
+		STACK_UNWIND (frame, local->op_ret, local->op_errno,
+			      &local->statvfs);
+	}
 
 	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+}
+
+int32_t
+map_single_lookup_cbk (call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       inode_t *inode,
+		       struct iatt *buf,
+                       dict_t *dict,
+                       struct iatt *postparent)
+{
+        call_frame_t *prev = NULL;
+        prev  = cookie;
+	
+	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
+
+	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, dict);
+
+	return 0;
+}
+
+int32_t
+map_root_lookup_cbk (call_frame_t *frame,
+                     void *cookie,
+                     xlator_t *this,
+                     int32_t op_ret,
+                     int32_t op_errno,
+                     inode_t *inode,
+                     struct iatt *buf,
+                     dict_t *dict,
+                     struct iatt *postparent)
+{
+	int          callcnt = 0;
+	map_local_t *local = NULL;
+	inode_t     *tmp_inode = NULL;
+	dict_t      *tmp_dict = NULL;
+
+	local = frame->local;
+	LOCK (&frame->lock);
+	{
+		callcnt = --local->call_count;
+		if ((op_ret == 0) && (local->op_ret == -1)) {
+			local->op_ret = 0;
+			local->stbuf = *buf;
+			if (dict)
+				local->dict = dict_ref (dict);
+			local->inode = inode_ref (inode);
+		}
+		if (op_ret == -1)
+			local->op_errno = op_errno;
+		
+	}
+	UNLOCK (&frame->lock);
+
+	if (!callcnt) {
+		tmp_dict = local->dict;
+		tmp_inode = local->inode;
+
+		STACK_UNWIND (frame, local->op_ret, 
+			      local->op_errno, local->inode, 
+			      &local->stbuf, local->dict);
+
+		inode_unref (local->inode);
+		if (tmp_dict)
+			dict_unref (tmp_dict);
+	}
 
 	return 0;
 }
 
 
 int32_t
-map_chmod (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   mode_t mode)
+map_opendir_cbk (call_frame_t *frame,
+		 void *cookie,
+		 xlator_t *this,
+		 int32_t op_ret,
+		 int32_t op_errno,
+		 fd_t *fd)
 {
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
+	int callcnt = 0;
+	map_local_t *local = NULL;
+	fd_t *local_fd = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+	local = frame->local;
+	LOCK (&frame->lock);
+	{
+		callcnt = --local->call_count;
 
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
+		if (op_ret == -1) {
+			local->op_errno = op_errno;
+			goto unlock;
+		}
+
+		local->op_ret = 0;
 	}
+ unlock:
+	UNLOCK (&frame->lock);
 
-	STACK_WIND (frame,
-		    default_chmod_cbk,
-		    subvol,
-		    subvol->fops->chmod,
-		    loc,
-		    mode);
+	if (!callcnt) {
+		local_fd = local->fd;
+		local->fd = NULL;
+
+		STACK_UNWIND (frame, local->op_ret, 
+			      local->op_errno, local_fd);
+
+		fd_unref (local_fd);
+	}
 	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+}
+
+int32_t
+map_single_readdir_cbk (call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			gf_dirent_t *entries)
+{
+        call_frame_t *prev = NULL;
+	gf_dirent_t  *orig_entry = NULL;
+
+        prev  = cookie;
 
+	list_for_each_entry (orig_entry, &entries->list, list) {
+		map_itransform (this, prev->this, orig_entry->d_ino, 
+				&orig_entry->d_ino);
+	}
+	STACK_UNWIND (frame, op_ret, op_errno, entries);
+	
 	return 0;
 }
 
+
 int32_t
-map_fchmod (call_frame_t *frame,
-	    xlator_t *this,
-	    fd_t *fd,
-	    mode_t mode)
+map_single_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+{
+        call_frame_t *prev = NULL;
+	gf_dirent_t  *orig_entry = NULL;
+
+        prev  = cookie;
+
+	list_for_each_entry (orig_entry, &entries->list, list) {
+		map_itransform (this, prev->this, orig_entry->d_ino,
+				&orig_entry->d_ino);
+                orig_entry->d_stat.ia_ino = orig_entry->d_ino;
+	}
+	STACK_UNWIND (frame, op_ret, op_errno, entries);
+	return 0;
+}
+
+int
+map_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int op_ret, int op_errno, gf_dirent_t *orig_entries);
+
+int
+map_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int op_ret, int op_errno, gf_dirent_t *orig_entries);
+
+int
+map_generic_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, gf_dirent_t *orig_entries,
+                         int whichop)
+{
+	map_local_t  *local = NULL;
+	gf_dirent_t   entries;
+	gf_dirent_t  *orig_entry = NULL;
+	gf_dirent_t  *entry = NULL;
+	call_frame_t *prev = NULL;
+	xlator_t     *subvol = NULL;
+	xlator_t     *next_subvol = NULL;
+        off_t         next_offset = 0;
+	int           count = 0;
+	fd_t         *local_fd = NULL;
+
+	INIT_LIST_HEAD (&entries.list);
+	prev = cookie;
+	local = frame->local;
+
+	if (op_ret < 0)
+		goto done;
+
+	list_for_each_entry (orig_entry, &orig_entries->list, list) {
+		subvol = prev->this;
+
+		entry = gf_dirent_for_name (orig_entry->d_name);
+		if (!entry) {
+			gf_log (this->name, GF_LOG_ERROR,
+				"memory allocation failed :(");
+			goto unwind;
+		}
+		
+		map_itransform (this, subvol, orig_entry->d_ino,
+				&entry->d_ino);
+		map_itransform (this, subvol, orig_entry->d_off,
+				&entry->d_off);
+		
+                if (whichop == GF_FOP_READDIRP)
+                        entry->d_stat.ia_ino = entry->d_ino;
+		entry->d_type = orig_entry->d_type;
+		entry->d_len  = orig_entry->d_len;
+		
+		list_add_tail (&entry->list, &entries.list);
+		count++;
+                next_offset = orig_entry->d_off;
+	}
+
+	op_ret = count;
+
+done:
+	if (count == 0) {
+                /* non-zero next_offset means that
+                   EOF is not yet hit on the current subvol
+                */
+                if (next_offset == 0) {
+                        next_subvol = map_subvol_next (this, prev->this);
+                } else {
+                        next_subvol = prev->this;
+                }
+
+		if (!next_subvol) {
+			goto unwind;
+		}
+
+                if (whichop == GF_FOP_READDIR)
+                        STACK_WIND (frame, map_readdir_cbk, next_subvol,
+                                    next_subvol->fops->readdir, local->fd,
+                                    local->size, 0);
+                else
+                        STACK_WIND (frame, map_readdirp_cbk, next_subvol,
+                                    next_subvol->fops->readdirp, local->fd,
+                                    local->size, 0);
+		return 0;
+	}
+
+unwind:
+	if (op_ret < 0)
+		op_ret = 0;
+
+	local_fd = local->fd;
+	local->fd = NULL;
+
+	STACK_UNWIND (frame, op_ret, op_errno, &entries);
+
+	fd_unref (local_fd);
+
+	gf_dirent_free (&entries);
+
+        return 0;
+}
+
+
+int
+map_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int op_ret, int op_errno, gf_dirent_t *orig_entries)
+{
+        map_generic_readdir_cbk (frame, cookie, this, op_ret, op_errno,
+                                 orig_entries, GF_FOP_READDIR);
+        return 0;
+}
+
+
+int
+map_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, gf_dirent_t *orig_entries)
+{
+        map_generic_readdir_cbk (frame, cookie, this, op_ret, op_errno,
+                                 orig_entries, GF_FOP_READDIRP);
+        return 0;
+}
+
+
+/* Management operations */
+
+static int32_t
+map_checksum_cbk (call_frame_t *frame,
+		      void *cookie,
+		      xlator_t *this,
+		      int32_t op_ret,
+		      int32_t op_errno,
+		      uint8_t *file_checksum,
+		      uint8_t *dir_checksum)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, file_checksum, dir_checksum);
+	return 0;
+}
+
+
+/* Fops starts here */
+
+int32_t
+map_stat (call_frame_t *frame,
+	  xlator_t *this,
+	  loc_t *loc)
 {
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
 
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
+	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
 	if (!subvol) {
 		op_errno = EINVAL;
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_fchmod_cbk,
-		    subvol,
-		    subvol->fops->fchmod,
-		    fd,
-		    mode);
+	STACK_WIND (frame, map_stat_cbk, subvol, subvol->fops->stat, loc);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -131,20 +927,21 @@ map_fchmod (call_frame_t *frame,
 }
 
 int32_t
-map_chown (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   uid_t uid,
-	   gid_t gid)
+map_setattr (call_frame_t *frame,
+             xlator_t *this,
+             loc_t *loc,
+             struct iatt *stbuf,
+             int32_t valid)
 {
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        GF_VALIDATE_OR_GOTO ("map", this, err);
+        GF_VALIDATE_OR_GOTO (this->name, frame, err);
+        GF_VALIDATE_OR_GOTO (this->name, loc, err);
+        GF_VALIDATE_OR_GOTO (this->name, loc->inode, err);
+        GF_VALIDATE_OR_GOTO (this->name, loc->path, err);
+        GF_VALIDATE_OR_GOTO (this->name, stbuf, err);
 
 	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
 	if (!subvol) {
@@ -152,13 +949,8 @@ map_chown (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_chown_cbk,
-		    subvol,
-		    subvol->fops->chown,
-		    loc,
-		    uid,
-		    gid);
+	STACK_WIND (frame, map_setattr_cbk, subvol,
+		    subvol->fops->setattr, loc, stbuf, valid);
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -167,19 +959,19 @@ map_chown (call_frame_t *frame,
 }
 
 int32_t
-map_fchown (call_frame_t *frame,
-	    xlator_t *this,
-	    fd_t *fd,
-	    uid_t uid,
-	    gid_t gid)
+map_fsetattr (call_frame_t *frame,
+              xlator_t *this,
+              fd_t *fd,
+              struct iatt *stbuf,
+              int32_t valid)
 {
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
+        GF_VALIDATE_OR_GOTO ("map", this, err);
+        GF_VALIDATE_OR_GOTO (this->name, frame, err);
+        GF_VALIDATE_OR_GOTO (this->name, fd, err);
+        GF_VALIDATE_OR_GOTO (this->name, stbuf, err);
 
 	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
 	if (!subvol) {
@@ -187,13 +979,8 @@ map_fchown (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_fchown_cbk,
-		    subvol,
-		    subvol->fops->fchown,
-		    fd,
-		    uid,
-		    gid);
+	STACK_WIND (frame, map_fsetattr_cbk, subvol,
+		    subvol->fops->fsetattr, fd, stbuf, valid);
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -222,12 +1009,9 @@ map_truncate (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_truncate_cbk,
-		    subvol,
-		    subvol->fops->truncate,
-		    loc,
-		    offset);
+	STACK_WIND (frame, map_truncate_cbk, subvol, 
+                    subvol->fops->truncate, loc, offset);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -255,47 +1039,10 @@ map_ftruncate (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_ftruncate_cbk,
-		    subvol,
-		    subvol->fops->ftruncate,
-		    fd,
-		    offset);
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+	STACK_WIND (frame, map_ftruncate_cbk, subvol,
+		    subvol->fops->ftruncate, fd, offset);
 
 	return 0;
-}
-
-int32_t
-map_utimens (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     struct timespec tv[2])
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame,
-		    default_utimens_cbk,
-		    subvol,
-		    subvol->fops->utimens,
-		    loc,
-		    tv);
-	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
 
@@ -323,12 +1070,9 @@ map_access (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_access_cbk,
-		    subvol,
-		    subvol->fops->access,
-		    loc,
-		    mask);
+	STACK_WIND (frame, map_access_cbk, subvol,
+		    subvol->fops->access, loc, mask);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -358,12 +1102,9 @@ map_readlink (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_readlink_cbk,
-		    subvol,
-		    subvol->fops->readlink,
-		    loc,
-		    size);
+	STACK_WIND (frame, map_readlink_cbk, subvol,
+		    subvol->fops->readlink, loc, size);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -391,11 +1132,8 @@ map_unlink (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_unlink_cbk,
-		    subvol,
-		    subvol->fops->unlink,
-		    loc);
+	STACK_WIND (frame, map_unlink_cbk, subvol, subvol->fops->unlink, loc);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -423,11 +1161,8 @@ map_rmdir (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_rmdir_cbk,
-		    subvol,
-		    subvol->fops->rmdir,
-		    loc);
+	STACK_WIND (frame, map_rmdir_cbk, subvol, subvol->fops->rmdir, loc);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -460,19 +1195,16 @@ map_rename (call_frame_t *frame,
 	}
 
 	if (newloc->path) {
-		new_subvol = get_mapping_subvol_from_path (this, 
-							   newloc->path);
+		new_subvol = get_mapping_subvol_from_path (this, newloc->path);
 		if (new_subvol && (new_subvol != old_subvol)) {
 			op_errno = EXDEV;
 			goto err;
 		}
 	}
 
-	STACK_WIND (frame,
-		    default_rename_cbk,
-		    old_subvol,
-		    old_subvol->fops->rename,
-		    oldloc, newloc);
+	STACK_WIND (frame, map_rename_cbk, old_subvol,
+		    old_subvol->fops->rename, oldloc, newloc);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -505,19 +1237,16 @@ map_link (call_frame_t *frame,
 	}
 
 	if (newloc->path) {
-		new_subvol = get_mapping_subvol_from_path (this, 
-							   newloc->path);
+		new_subvol = get_mapping_subvol_from_path (this, newloc->path);
 		if (new_subvol && (new_subvol != old_subvol)) {
 			op_errno = EXDEV;
 			goto err;
 		}
 	}
 
-	STACK_WIND (frame,
-		    default_link_cbk,
-		    old_subvol,
-		    old_subvol->fops->link,
-		    oldloc, newloc);
+	STACK_WIND (frame, map_link_cbk, old_subvol,
+		    old_subvol->fops->link, oldloc, newloc);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -530,7 +1259,7 @@ int32_t
 map_open (call_frame_t *frame,
 	  xlator_t *this,
 	  loc_t *loc,
-	  int32_t flags, fd_t *fd)
+	  int32_t flags, fd_t *fd, int wbflags)
 {
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
@@ -547,11 +1276,9 @@ map_open (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_open_cbk,
-		    subvol,
-		    subvol->fops->open,
-		    loc, flags, fd);
+	STACK_WIND (frame, map_open_cbk, subvol, 
+                    subvol->fops->open, loc, flags, fd, wbflags);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -580,16 +1307,12 @@ map_readv (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_readv_cbk,
-		    subvol,
-		    subvol->fops->readv,
-		    fd,
-		    size,
-		    offset);
+	STACK_WIND (frame, map_readv_cbk, subvol,
+		    subvol->fops->readv, fd, size, offset);
+
 	return 0;
  err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+	STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL, NULL);
 
 	return 0;
 }
@@ -600,7 +1323,8 @@ map_writev (call_frame_t *frame,
 	    fd_t *fd,
 	    struct iovec *vector,
 	    int32_t count,
-	    off_t off)
+	    off_t off,
+            struct iobref *iobref)
 {
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
@@ -616,14 +1340,9 @@ map_writev (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_writev_cbk,
-		    subvol,
-		    subvol->fops->writev,
-		    fd,
-		    vector,
-		    count,
-		    off);
+	STACK_WIND (frame, map_writev_cbk, subvol,
+		    subvol->fops->writev, fd, vector, count, off, iobref);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -650,11 +1369,8 @@ map_flush (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_flush_cbk,
-		    subvol,
-		    subvol->fops->flush,
-		    fd);
+	STACK_WIND (frame, map_flush_cbk, subvol, subvol->fops->flush, fd);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -683,12 +1399,9 @@ map_fsync (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_fsync_cbk,
-		    subvol,
-		    subvol->fops->fsync,
-		    fd,
-		    flags);
+	STACK_WIND (frame, map_fsync_cbk, subvol,
+		    subvol->fops->fsync, fd, flags);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -715,11 +1428,8 @@ map_fstat (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_fstat_cbk,
-		    subvol,
-		    subvol->fops->fstat,
-		    fd);
+	STACK_WIND (frame, map_fstat_cbk, subvol, subvol->fops->fstat, fd);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -749,14 +1459,9 @@ map_getdents (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_getdents_cbk,
-		    subvol,
-		    subvol->fops->getdents,
-		    fd,
-		    size,
-		    offset,
-		    flag);
+	STACK_WIND (frame, map_getdents_cbk, subvol,
+		    subvol->fops->getdents, fd, size, offset, flag);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -786,14 +1491,9 @@ map_setdents (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_setdents_cbk,
-		    subvol,
-		    subvol->fops->setdents,
-		    fd,
-		    flags,
-		    entries,
-		    count);
+	STACK_WIND (frame, map_setdents_cbk, subvol,
+		    subvol->fops->setdents, fd, flags, entries, count);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -822,12 +1522,9 @@ map_fsyncdir (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_fsyncdir_cbk,
-		    subvol,
-		    subvol->fops->fsyncdir,
-		    fd,
-		    flags);
+	STACK_WIND (frame, map_fsyncdir_cbk, subvol,
+		    subvol->fops->fsyncdir, fd, flags);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -845,7 +1542,6 @@ map_setxattr (call_frame_t *frame,
 	      dict_t *dict,
 	      int32_t flags)
 {
-	/* TODO: support for 'get' 'put' API */
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
 
@@ -861,13 +1557,9 @@ map_setxattr (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_setxattr_cbk,
-		    subvol,
-		    subvol->fops->setxattr,
-		    loc,
-		    dict,
-		    flags);
+	STACK_WIND (frame, map_setxattr_cbk, subvol,
+		    subvol->fops->setxattr, loc, dict, flags);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -881,7 +1573,6 @@ map_getxattr (call_frame_t *frame,
 	      loc_t *loc,
 	      const char *name)
 {
-	/* TODO: support for 'get' 'put' API */
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
 
@@ -897,12 +1588,41 @@ map_getxattr (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_getxattr_cbk,
-		    subvol,
-		    subvol->fops->getxattr,
-		    loc,
-		    name);
+	STACK_WIND (frame, map_getxattr_cbk, subvol,
+		    subvol->fops->getxattr, loc, name);
+
+	return 0;
+ err:
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+
+	return 0;
+}
+
+
+int32_t
+map_fsetxattr (call_frame_t *frame,
+               xlator_t *this,
+               fd_t *fd,
+               dict_t *dict,
+               int32_t flags)
+{
+	int32_t   op_errno = 1;
+	xlator_t *subvol   = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
+	if (!subvol) {
+		op_errno = EINVAL;
+		goto err;
+	}
+
+	STACK_WIND (frame, map_fsetxattr_cbk, subvol, 
+                    subvol->fops->fsetxattr, fd, dict, flags);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -911,6 +1631,37 @@ map_getxattr (call_frame_t *frame,
 }
 
 int32_t
+map_fgetxattr (call_frame_t *frame,
+               xlator_t *this,
+               fd_t *fd,
+               const char *name)
+{
+	int32_t   op_errno = 1;
+	xlator_t *subvol   = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
+	if (!subvol) {
+		op_errno = EINVAL;
+		goto err;
+	}
+
+	STACK_WIND (frame, map_fgetxattr_cbk, subvol,
+		    subvol->fops->fgetxattr, fd, name);
+
+	return 0;
+ err:
+	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+
+	return 0;
+}
+
+
+int32_t
 map_xattrop (call_frame_t *frame,
 	     xlator_t *this,
 	     loc_t *loc,
@@ -932,13 +1683,9 @@ map_xattrop (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_xattrop_cbk,
-		    subvol,
-		    subvol->fops->xattrop,
-		    loc,
-		    flags,
-		    dict);
+	STACK_WIND (frame, map_xattrop_cbk, subvol,
+		    subvol->fops->xattrop, loc, flags, dict);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -967,14 +1714,10 @@ map_fxattrop (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_fxattrop_cbk,
-		    subvol,
-		    subvol->fops->fxattrop,
-		    fd,
-		    flags,
-		    dict);
-	return 0;
+	STACK_WIND (frame, map_fxattrop_cbk, subvol,
+		    subvol->fops->fxattrop, fd, flags, dict);
+
+        return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
 
@@ -1002,12 +1745,9 @@ map_removexattr (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_removexattr_cbk,
-		    subvol,
-		    subvol->fops->removexattr,
-		    loc,
-		    name);
+	STACK_WIND (frame, map_removexattr_cbk, subvol,
+		    subvol->fops->removexattr, loc, name);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1020,7 +1760,7 @@ map_lk (call_frame_t *frame,
 	xlator_t *this,
 	fd_t *fd,
 	int32_t cmd,
-	struct flock *lock)
+	struct gf_flock *lock)
 {
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
@@ -1036,13 +1776,9 @@ map_lk (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_lk_cbk,
-		    subvol,
-		    subvol->fops->lk,
-		    fd,
-		    cmd,
-		    lock);
+	STACK_WIND (frame, map_lk_cbk, subvol,
+		    subvol->fops->lk, fd, cmd, lock);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1053,7 +1789,7 @@ map_lk (call_frame_t *frame,
 
 int32_t
 map_inodelk (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, int32_t cmd, struct flock *lock)
+	     const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
 {
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
@@ -1070,11 +1806,9 @@ map_inodelk (call_frame_t *frame, xlator_t *this,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_inodelk_cbk,
-		    subvol,
-		    subvol->fops->inodelk,
-		    loc, cmd, lock);
+	STACK_WIND (frame, map_inodelk_cbk, subvol,
+		    subvol->fops->inodelk, volume, loc, cmd, lock);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1085,7 +1819,7 @@ map_inodelk (call_frame_t *frame, xlator_t *this,
 
 int32_t
 map_finodelk (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, int32_t cmd, struct flock *lock)
+	      const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
 {
 	int32_t op_errno = 1;
 	xlator_t *subvol   = NULL;
@@ -1101,11 +1835,9 @@ map_finodelk (call_frame_t *frame, xlator_t *this,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_finodelk_cbk,
-		    subvol,
-		    subvol->fops->finodelk,
-		    fd, cmd, lock);
+	STACK_WIND (frame, map_finodelk_cbk, subvol,
+		    subvol->fops->finodelk, volume, fd, cmd, lock);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1115,7 +1847,7 @@ map_finodelk (call_frame_t *frame, xlator_t *this,
 
 int32_t
 map_entrylk (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, const char *basename,
+	     const char *volume, loc_t *loc, const char *basename,
 	     entrylk_cmd cmd, entrylk_type type)
 {
 	int32_t op_errno = 1;
@@ -1133,10 +1865,9 @@ map_entrylk (call_frame_t *frame, xlator_t *this,
 		goto err;
 	}
 
-	STACK_WIND (frame, default_entrylk_cbk,
-		    subvol,
-		    subvol->fops->entrylk,
-		    loc, basename, cmd, type);
+	STACK_WIND (frame, map_entrylk_cbk, subvol,
+		    subvol->fops->entrylk, volume, loc, basename, cmd, type);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1146,7 +1877,7 @@ map_entrylk (call_frame_t *frame, xlator_t *this,
 
 int32_t
 map_fentrylk (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, const char *basename,
+	      const char *volume, fd_t *fd, const char *basename,
 	      entrylk_cmd cmd, entrylk_type type)
 {
 	int32_t op_errno = 1;
@@ -1163,10 +1894,9 @@ map_fentrylk (call_frame_t *frame, xlator_t *this,
 		goto err;
 	}
 
-	STACK_WIND (frame, default_fentrylk_cbk,
-		    subvol,
-		    subvol->fops->fentrylk,
-		    fd, basename, cmd, type);
+	STACK_WIND (frame, map_fentrylk_cbk, subvol,
+		    subvol->fops->fentrylk, volume, fd, basename, cmd, type);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1195,12 +1925,9 @@ map_checksum (call_frame_t *frame,
 		goto err;
 	}
 
-	STACK_WIND (frame,
-		    default_checksum_cbk,
-		    subvol,
-		    subvol->fops->checksum,
-		    loc,
-		    flag);
+	STACK_WIND (frame, map_checksum_cbk, subvol,
+		    subvol->fops->checksum, loc, flag);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1208,25 +1935,6 @@ map_checksum (call_frame_t *frame,
 	return 0;
 }
 
-static int32_t
-map_newentry_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-		  struct stat *buf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->st_ino, &buf->st_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-
-}
-
 
 int32_t
 map_mknod (call_frame_t *frame,
@@ -1257,11 +1965,9 @@ map_mknod (call_frame_t *frame,
 			loc->path);
 	}
 
-	STACK_WIND (frame,
-		    map_newentry_cbk,
-		    subvol,
-		    subvol->fops->mknod,
-		    loc, mode, rdev);
+	STACK_WIND (frame, map_newentry_cbk, subvol,
+		    subvol->fops->mknod, loc, mode, rdev);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1297,11 +2003,8 @@ map_mkdir (call_frame_t *frame,
 			loc->path);
 	}
 
-	STACK_WIND (frame,
-		    map_newentry_cbk,
-		    subvol,
-		    subvol->fops->mkdir,
-		    loc, mode);
+	STACK_WIND (frame, map_newentry_cbk, subvol,
+		    subvol->fops->mkdir, loc, mode);
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1338,11 +2041,9 @@ map_symlink (call_frame_t *frame,
 			loc->path);
 	}
 
-	STACK_WIND (frame,
-		    map_newentry_cbk,
-		    subvol,
-		    subvol->fops->symlink,
-		    linkpath, loc);
+	STACK_WIND (frame, map_newentry_cbk, subvol,
+		    subvol->fops->symlink, linkpath, loc);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1350,25 +2051,6 @@ map_symlink (call_frame_t *frame,
 	return 0;
 }
 
-
-static int32_t
-map_create_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd,
-		inode_t *inode,
-		struct stat *buf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->st_ino, &buf->st_ino);
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
-	return 0;
-}
-
 int32_t
 map_create (call_frame_t *frame,
 	    xlator_t *this,
@@ -1398,10 +2080,9 @@ map_create (call_frame_t *frame,
 			loc->path);
 	}
 
-	STACK_WIND (frame, map_create_cbk,
-		    subvol,
-		    subvol->fops->create,
-		    loc, flags, mode, fd);
+	STACK_WIND (frame, map_create_cbk, subvol,
+		    subvol->fops->create, loc, flags, mode, fd);
+
 	return 0;
  err:
 	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
@@ -1409,73 +2090,6 @@ map_create (call_frame_t *frame,
 	return 0;
 }
 
-int32_t
-map_single_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct stat *buf,
-		       dict_t *dict)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->st_ino, &buf->st_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, dict);
-
-	return 0;
-}
-
-int32_t
-map_lookup_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-		struct stat *buf,
-		dict_t *dict)
-{
-	int callcnt = 0;
-	map_local_t *local = NULL;
-	inode_t *tmp_inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	local = frame->local;
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if ((op_ret == 0) && (local->op_ret == -1)) {
-			local->op_ret = 0;
-			local->stbuf = *buf;
-			if (dict)
-				local->dict = dict_ref (dict);
-			local->inode = inode_ref (inode);
-		}
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-		
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		tmp_dict = local->dict;
-		tmp_inode = local->inode;
-
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local->inode, 
-			      &local->stbuf, local->dict);
-
-		inode_unref (local->inode);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
 
 int32_t
 map_lookup (call_frame_t *frame,
@@ -1517,17 +2131,14 @@ map_lookup (call_frame_t *frame,
 	}
 
 	/* Just one callback */
-	STACK_WIND (frame,
-		    map_single_lookup_cbk,
-		    subvol,
-		    subvol->fops->lookup,
-		    loc,
-		    xattr_req);
+	STACK_WIND (frame, map_single_lookup_cbk, subvol,
+		    subvol->fops->lookup, loc, xattr_req);
 
 	return 0;
 
  root_inode:
-	local = CALLOC (1, sizeof (map_local_t));
+	local = GF_CALLOC (1, sizeof (map_local_t),
+                           gf_map_mt_map_local_t);
 
 	frame->local = local;
 	local->call_count = priv->child_count;
@@ -1535,12 +2146,8 @@ map_lookup (call_frame_t *frame,
 
 	trav = this->children;
 	while (trav) {
-		STACK_WIND (frame,
-			    map_lookup_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->lookup,
-			    loc,
-			    xattr_req);
+		STACK_WIND (frame, map_root_lookup_cbk, trav->xlator,
+			    trav->xlator->fops->lookup, loc, xattr_req);
 		trav = trav->next;
 	}
 
@@ -1551,93 +2158,7 @@ map_lookup (call_frame_t *frame,
 
 	return 0;
 }
-/*
- * unify_normalize_stats -
- */
-void
-map_normalize_stats (struct statvfs *buf,
-		     unsigned long bsize,
-		     unsigned long frsize)
-{
-	double factor;
-
-	if (buf->f_bsize != bsize) {
-		factor = ((double) buf->f_bsize) / bsize;
-		buf->f_bsize  = bsize;
-		buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-		buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
-	}
-  
-	if (buf->f_frsize != frsize) {
-		factor = ((double) buf->f_frsize) / frsize;
-		buf->f_frsize = frsize;
-		buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-	}
-}
-
-
-int32_t
-map_statfs_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		struct statvfs *stbuf)
-{
-	struct statvfs *dict_buf = NULL;
-	map_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	unsigned long bsize;
-	unsigned long frsize;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		this_call_cnt = --local->call_count;
-
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-		local->op_ret = 0;
-		
-		/* when a call is successfull, add it to local->dict */
-		dict_buf = &local->statvfs;
-		
-		if (dict_buf->f_bsize != 0) {
-			bsize  = max (dict_buf->f_bsize, 
-				      stbuf->f_bsize);
-			
-			frsize = max (dict_buf->f_frsize, 
-				      stbuf->f_frsize);
-			map_normalize_stats(dict_buf, bsize, frsize);
-			map_normalize_stats(stbuf, bsize, frsize);
-		} else {
-			dict_buf->f_bsize   = stbuf->f_bsize;
-			dict_buf->f_frsize  = stbuf->f_frsize;
-		}
-		
-		dict_buf->f_blocks += stbuf->f_blocks;
-		dict_buf->f_bfree  += stbuf->f_bfree;
-		dict_buf->f_bavail += stbuf->f_bavail;
-		dict_buf->f_files  += stbuf->f_files;
-		dict_buf->f_ffree  += stbuf->f_ffree;
-		dict_buf->f_favail += stbuf->f_favail;
-		dict_buf->f_fsid    = stbuf->f_fsid;
-		dict_buf->f_flag    = stbuf->f_flag;
-		dict_buf->f_namemax = stbuf->f_namemax;
-	}
-unlock:
-	UNLOCK (&frame->lock);
 
-	if (!this_call_cnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			      &local->statvfs);
-	}
-
-	return 0;
-}
 
 int32_t
 map_statfs (call_frame_t *frame,
@@ -1664,16 +2185,13 @@ map_statfs (call_frame_t *frame,
 	}
 	
 	/* Just one callback */
-	STACK_WIND (frame,
-		    default_statfs_cbk,
-		    subvol,
-		    subvol->fops->statfs,
-		    loc);
+	STACK_WIND (frame, map_statfs_cbk, subvol, subvol->fops->statfs, loc);
 
 	return 0;
 
  root_inode:
-	local = CALLOC (1, sizeof (map_local_t));
+	local = GF_CALLOC (1, sizeof (map_local_t),
+                           gf_map_mt_map_local_t);
 
 	priv = this->private;
 	frame->local = local;
@@ -1682,11 +2200,8 @@ map_statfs (call_frame_t *frame,
 
 	trav = this->children;
 	while (trav) {
-		STACK_WIND (frame,
-			    map_statfs_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->statfs,
-			    loc);
+		STACK_WIND (frame, map_statfs_cbk, trav->xlator,
+			    trav->xlator->fops->statfs, loc);
 		trav = trav->next;
 	}
 
@@ -1698,46 +2213,6 @@ map_statfs (call_frame_t *frame,
 }
 
 int32_t
-map_opendir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 fd_t *fd)
-{
-	int callcnt = 0;
-	map_local_t *local = NULL;
-	fd_t *local_fd = NULL;
-
-	local = frame->local;
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-
-		local->op_ret = 0;
-	}
- unlock:
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local_fd = local->fd;
-		local->fd = NULL;
-
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local_fd);
-
-		fd_unref (local_fd);
-	}
-	return 0;
-}
-
-
-int32_t
 map_opendir (call_frame_t *frame,
 	     xlator_t *this,
 	     loc_t *loc, fd_t *fd)
@@ -1762,15 +2237,14 @@ map_opendir (call_frame_t *frame,
 	}
 	
 	/* Just one callback */
-	STACK_WIND (frame,
-		    default_opendir_cbk,
-		    subvol,
-		    subvol->fops->opendir,
-		    loc, fd);
+	STACK_WIND (frame, map_opendir_cbk, subvol,
+		    subvol->fops->opendir, loc, fd);
+
 	return 0;
 
  root_inode:
-	local = CALLOC (1, sizeof (map_local_t));
+	local = GF_CALLOC (1, sizeof (map_local_t),
+                           gf_map_mt_map_local_t);
 
 	priv = this->private;
 	frame->local = local;
@@ -1780,11 +2254,8 @@ map_opendir (call_frame_t *frame,
 
 	trav = this->children;
 	while (trav) {
-		STACK_WIND (frame,
-			    map_opendir_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->opendir,
-			    loc, fd);
+		STACK_WIND (frame, map_opendir_cbk, trav->xlator,
+			    trav->xlator->fops->opendir, loc, fd);
 		trav = trav->next;
 	}
 	
@@ -1797,109 +2268,8 @@ map_opendir (call_frame_t *frame,
 
 
 int32_t
-map_single_readdir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			gf_dirent_t *entries)
-{
-        call_frame_t *prev = NULL;
-	gf_dirent_t  *orig_entry = NULL;
-
-        prev  = cookie;
-
-	list_for_each_entry (orig_entry, &entries->list, list) {
-		map_itransform (this, prev->this, orig_entry->d_ino, 
-				&orig_entry->d_ino);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, entries);
-	
-	return 0;
-}
-
-
-int
-map_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, gf_dirent_t *orig_entries)
-{
-	map_local_t  *local = NULL;
-	gf_dirent_t   entries;
-	gf_dirent_t  *orig_entry = NULL;
-	gf_dirent_t  *entry = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *subvol = NULL;
-	xlator_t     *next = NULL;
-	int           count = 0;
-	fd_t         *local_fd = NULL;
-
-	INIT_LIST_HEAD (&entries.list);
-	prev = cookie;
-	local = frame->local;
-
-	if (op_ret < 0)
-		goto done;
-
-	list_for_each_entry (orig_entry, &orig_entries->list, list) {
-		subvol = prev->this;
-
-		entry = gf_dirent_for_name (orig_entry->d_name);
-		if (!entry) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"memory allocation failed :(");
-			goto unwind;
-		}
-		
-		map_itransform (this, subvol, orig_entry->d_ino,
-				&entry->d_ino);
-		map_itransform (this, subvol, orig_entry->d_off,
-				&entry->d_off);
-		
-		entry->d_type = orig_entry->d_type;
-		entry->d_len  = orig_entry->d_len;
-		
-		list_add_tail (&entry->list, &entries.list);
-		count++;
-	}
-
-	op_ret = count;
-
-done:
-	if (count == 0) {
-		next = map_subvol_next (this, prev->this);
-		if (!next) {
-			goto unwind;
-		}
-
-		STACK_WIND (frame, map_readdir_cbk,
-			    next, next->fops->readdir,
-			    local->fd, local->size, 0);
-		return 0;
-	}
-
-unwind:
-	if (op_ret < 0)
-		op_ret = 0;
-
-	local_fd = local->fd;
-	local->fd = NULL;
-
-	STACK_UNWIND (frame, op_ret, op_errno, &entries);
-
-	fd_unref (local_fd);
-
-	gf_dirent_free (&entries);
-
-        return 0;
-}
-
-
-int32_t
-map_readdir (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     size_t size,
-	     off_t yoff)
+map_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t yoff, int whichop)
 {
 	int32_t        op_errno = EINVAL;
 	xlator_t      *subvol = NULL;
@@ -1922,17 +2292,19 @@ map_readdir (call_frame_t *frame,
 	}
 	
 	/* Just one callback */
-	
-	STACK_WIND (frame,
-		    map_single_readdir_cbk,
-		    subvol,
-		    subvol->fops->readdir,
-		    fd, size, yoff);
+        if (whichop == GF_FOP_READDIR)
+                STACK_WIND (frame, map_single_readdir_cbk, subvol,
+                            subvol->fops->readdir, fd, size, yoff);
+        else
+                STACK_WIND (frame, map_single_readdirp_cbk, subvol,
+                            subvol->fops->readdirp, fd, size, yoff);
+
 	return 0;
 
  root_inode:
 	/* readdir on '/' */
-	local = CALLOC (1, sizeof (map_local_t));
+	local = GF_CALLOC (1, sizeof (map_local_t),
+                           gf_map_mt_map_local_t);
 	if (!local) {
 		gf_log (this->name, GF_LOG_ERROR,
 			"memory allocation failed :(");
@@ -1950,9 +2322,12 @@ map_readdir (call_frame_t *frame,
 
 	map_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
 
-	STACK_WIND (frame, map_readdir_cbk,
-		    xvol, xvol->fops->readdir,
-		    fd, size, xoff);
+        if (whichop == GF_FOP_READDIR)
+                STACK_WIND (frame, map_readdir_cbk, xvol, xvol->fops->readdir,
+                            fd, size, xoff);
+        else
+                STACK_WIND (frame, map_readdirp_cbk, xvol, xvol->fops->readdirp,
+                            fd, size, xoff);
 
 	return 0;
  err:
@@ -1962,42 +2337,22 @@ map_readdir (call_frame_t *frame,
 }
 
 
-#if 0 
-/* TODO : do it later as currently only unify uses this mop and mostly 
-   unify will be used below map  */
+
 int32_t
-map_stats_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct xlator_stats *stats)
+map_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t yoff)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, stats);
-	return 0;
+        map_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIR);
+        return 0;
 }
 
-
 int32_t
-map_stats (call_frame_t *frame,
-	   xlator_t *this,
-	   int32_t flags)
+map_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t yoff)
 {
-	STACK_WIND (frame,
-		    map_stats_cbk,
-		    subvol,
-		    subvol->mops->stats,
-		    flags);
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
+        map_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP);
+        return 0;
 }
-#endif /* if 0 */
-
-
-/* TODO: define the behavior of notify */
 
 
 void
@@ -2010,22 +2365,40 @@ fini (xlator_t *this)
 	priv = this->private;
 
 	if (priv) {
-		if (priv->xlarray)
-			FREE (priv->xlarray);
+		GF_FREE (priv->xlarray);
 
 		trav_map = priv->map;
 		while (trav_map) {
 			tmp_map = trav_map;
 			trav_map = trav_map->next;
-			FREE (tmp_map);
+			GF_FREE (tmp_map);
 		}
 
-		FREE(priv);
+		GF_FREE(priv);
 	}
 
 	return;
 }
 
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_map_mt_end + 1);
+        
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                                "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
 int
 init (xlator_t *this)
 {
@@ -2040,7 +2413,8 @@ init (xlator_t *this)
 	char *dup_map_pair = NULL;
 	char *dir_str = NULL;
 	char *subvol_str = NULL;
-	char *default_xl = NULL;
+	char *map_xl = NULL;
+
 
 	if (!this->children) {
 		gf_log (this->name,GF_LOG_ERROR,
@@ -2053,7 +2427,8 @@ init (xlator_t *this)
 			"dangling volume. check volfile ");
 	}
 
-	priv = CALLOC (1, sizeof (map_private_t));
+	priv = GF_CALLOC (1, sizeof (map_private_t),
+                          gf_map_mt_map_private_t);
 	this->private = priv;
 
 	/* allocate xlator array */
@@ -2062,7 +2437,8 @@ init (xlator_t *this)
 		count++;
 		trav = trav->next;
 	}
-	priv->xlarray = CALLOC (1, sizeof (struct map_xlator_array) * count);
+	priv->xlarray = GF_CALLOC (1, sizeof (struct map_xlator_array) * count,
+                                   gf_map_mt_map_xlator_array);
 	priv->child_count = count;
 
 	/* build xlator array */
@@ -2082,7 +2458,7 @@ init (xlator_t *this)
 	}
 	map_pair_str = strtok_r (pattern_string, ";", &tmp_str);
 	while (map_pair_str) {
-		dup_map_pair = strdup (map_pair_str);
+		dup_map_pair = gf_strdup (map_pair_str);
 		dir_str = strtok_r (dup_map_pair, ":", &tmp_str1);
 		if (!dir_str) {
 			gf_log (this->name, GF_LOG_ERROR, 
@@ -2104,15 +2480,15 @@ init (xlator_t *this)
 			goto err;
 		}
 		
-		FREE (dup_map_pair);
+		GF_FREE (dup_map_pair);
 
 		map_pair_str = strtok_r (NULL, ";", &tmp_str);
 	}
 
 	/* default-volume brick4 */
-	ret = dict_get_str (this->options, "default-volume", &default_xl);
+	ret = dict_get_str (this->options, "default-volume", &map_xl);
 	if (ret == 0) {
-		ret = assign_default_subvol (this, default_xl);
+		ret = assign_default_subvol (this, map_xl);
 		if (ret != 0) {
 			gf_log (this->name, GF_LOG_ERROR, 
 				"assigning default failed");
@@ -2135,18 +2511,15 @@ struct xlator_fops fops = {
 	.create      = map_create,
 
 	.stat        = map_stat,
-	.chmod       = map_chmod,
-	.chown       = map_chown,
-	.fchown      = map_fchown,
-	.fchmod      = map_fchmod,
 	.fstat       = map_fstat,
-	.utimens     = map_utimens,
 	.truncate    = map_truncate,
 	.ftruncate   = map_ftruncate,
 	.access      = map_access,
 	.readlink    = map_readlink,
 	.setxattr    = map_setxattr,
 	.getxattr    = map_getxattr,
+	.fsetxattr   = map_fsetxattr,
+	.fgetxattr   = map_fgetxattr,
 	.removexattr = map_removexattr,
 	.open        = map_open,
 	.readv       = map_readv,
@@ -2157,6 +2530,7 @@ struct xlator_fops fops = {
 	.lk          = map_lk,
 	.opendir     = map_opendir,
 	.readdir     = map_readdir,
+	.readdirp    = map_readdirp,
 	.fsyncdir    = map_fsyncdir,
 	.symlink     = map_symlink,
 	.unlink      = map_unlink,
@@ -2173,9 +2547,8 @@ struct xlator_fops fops = {
 	.setdents    = map_setdents,
 	.getdents    = map_getdents,
 	.checksum    = map_checksum,
-};
-
-struct xlator_mops mops = {
+        .setattr     = map_setattr,
+        .fsetattr    = map_fsetattr,
 };
 
 struct xlator_cbks cbks = {
diff --git a/xlators/cluster/map/src/map.h b/xlators/cluster/map/src/map.h
index 0f1aabfd6..7703a543e 100644
--- a/xlators/cluster/map/src/map.h
+++ b/xlators/cluster/map/src/map.h
@@ -1,26 +1,17 @@
 /*
-  Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MAP_H__
 #define __MAP_H__
 
 #include "xlator.h"
+#include "map-mem-types.h"
 
 struct map_pattern {
 	struct map_pattern *next;
@@ -46,7 +37,7 @@ typedef struct {
 	int32_t        op_errno;
 	int            call_count;
 	struct statvfs statvfs;
-	struct stat    stbuf;
+	struct iatt    stbuf;
 	inode_t       *inode;
 	dict_t        *dict;
 	fd_t          *fd;
diff --git a/xlators/cluster/stripe/src/Makefile.am b/xlators/cluster/stripe/src/Makefile.am
index 60e0a1568..2d151422a 100644
--- a/xlators/cluster/stripe/src/Makefile.am
+++ b/xlators/cluster/stripe/src/Makefile.am
@@ -2,13 +2,19 @@
 xlator_LTLIBRARIES = stripe.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-stripe_la_LDFLAGS = -module -avoidversion
+stripe_la_LDFLAGS = -module -avoid-version
+
+stripe_la_SOURCES = stripe.c stripe-helpers.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-stripe_la_SOURCES = stripe.c
 stripe_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = stripe.h stripe-mem-types.h $(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/stripe/src/stripe-helpers.c b/xlators/cluster/stripe/src/stripe-helpers.c
new file mode 100644
index 000000000..a83abdc72
--- /dev/null
+++ b/xlators/cluster/stripe/src/stripe-helpers.c
@@ -0,0 +1,675 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <fnmatch.h>
+
+#include "stripe.h"
+#include "byte-order.h"
+#include "mem-types.h"
+
+void
+stripe_local_wipe (stripe_local_t *local)
+{
+        if (!local)
+                goto out;
+
+        loc_wipe (&local->loc);
+        loc_wipe (&local->loc2);
+
+        if (local->fd)
+                fd_unref (local->fd);
+
+        if (local->inode)
+                inode_unref (local->inode);
+
+        if (local->xattr)
+                dict_unref (local->xattr);
+
+        if (local->xdata)
+                dict_unref (local->xdata);
+
+out:
+        return;
+}
+
+
+
+int
+stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
+{
+        dict_t  *dst  = NULL;
+        int64_t *ptr  = 0, *size = NULL;
+        int32_t  ret  = -1;
+
+        dst = data;
+
+        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
+                ret = dict_get_bin (dst, key, (void **)&size);
+                if (ret < 0) {
+                        size = GF_CALLOC (1, sizeof (int64_t),
+                                          gf_common_mt_char);
+                        if (size == NULL) {
+                                gf_log ("stripe", GF_LOG_WARNING,
+                                        "memory allocation failed");
+                                goto out;
+                        }
+                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
+                        if (ret < 0) {
+                                gf_log ("stripe", GF_LOG_WARNING,
+                                        "stripe aggregate dict set failed");
+                                GF_FREE (size);
+                                goto out;
+                        }
+                }
+
+                ptr = data_to_bin (value);
+                if (ptr == NULL) {
+                        gf_log ("stripe", GF_LOG_WARNING, "data to bin failed");
+                        goto out;
+                }
+
+                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+        } else if (strcmp (key, GF_CONTENT_KEY)) {
+                /* No need to aggregate 'CONTENT' data */
+                ret = dict_set (dst, key, value);
+                if (ret)
+                        gf_log ("stripe", GF_LOG_WARNING, "xattr dict set failed");
+        }
+
+out:
+        return 0;
+}
+
+
+void
+stripe_aggregate_xattr (dict_t *dst, dict_t *src)
+{
+        if ((dst == NULL) || (src == NULL)) {
+                goto out;
+        }
+
+        dict_foreach (src, stripe_aggregate, dst);
+out:
+        return;
+}
+
+
+int32_t
+stripe_xattr_aggregate (char *buffer, stripe_local_t *local, int32_t *total)
+{
+        int32_t              i     = 0;
+        int32_t              ret   = -1;
+        int32_t              len   = 0;
+        char                *sbuf  = NULL;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (!buffer || !local || !local->xattr_list)
+                goto out;
+
+        sbuf = buffer;
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+                len = xattr->xattr_len;
+
+                if (len && xattr && xattr->xattr_value) {
+                        memcpy (buffer, xattr->xattr_value, len);
+                        buffer += len;
+                        *buffer++ = ' ';
+                }
+        }
+
+        *--buffer = '\0';
+        if (total)
+                *total = buffer - sbuf;
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int32_t
+stripe_free_xattr_str (stripe_local_t *local)
+{
+        int32_t              i     = 0;
+        int32_t              ret   = -1;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (!local || !local->xattr_list)
+                goto out;
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+
+                if (xattr && xattr->xattr_value)
+                        GF_FREE (xattr->xattr_value);
+        }
+
+        ret = 0;
+ out:
+        return ret;
+}
+
+
+int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz)
+{
+        int32_t              ret   = -1, i = 0, len = 0;
+        dict_t              *tmp1  = NULL, *tmp2 = NULL;
+        char                *buf   = NULL;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (xattr_serz == NULL) {
+                goto out;
+        }
+
+        tmp2 = dict_new ();
+
+        if (tmp2 == NULL) {
+                goto out;
+        }
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+                len = xattr->xattr_len;
+
+                if (len && xattr && xattr->xattr_value) {
+                        ret = dict_reset (tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "dict_reset failed (%s)",
+                                        strerror (-ret));
+                        }
+
+                        ret = dict_unserialize (xattr->xattr_value,
+                                                xattr->xattr_len,
+                                                &tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_unserialize failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+
+                        tmp1 = dict_copy (tmp2, tmp1);
+                        if (tmp1 == NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_copy failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        len = dict_serialized_length (tmp1);
+        if (len > 0) {
+                buf = GF_CALLOC (1, len, gf_common_mt_dict_t);
+                if (buf == NULL) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_serialize (tmp1, buf);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "dict_serialize failed (%s)", strerror (-ret));
+                        ret = -1;
+                        goto out;
+                }
+
+                *xattr_serz = buf;
+        }
+
+        ret = 0;
+out:
+        if (tmp1 != NULL) {
+                dict_unref (tmp1);
+        }
+
+        if (tmp2 != NULL) {
+                dict_unref (tmp2);
+        }
+
+        return ret;
+}
+
+
+int32_t
+stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            char **xattr_serz)
+{
+        int      ret             = -1;
+        int32_t  padding         = 0;
+        int32_t  tlen            = 0;
+        char stripe_size_str[20] = {0,};
+        char    *pathinfo_serz   = NULL;
+
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
+                goto out;
+        }
+
+        (void) snprintf (stripe_size_str, 20, "%ld",
+                         (local->fctx) ? local->fctx->stripe_size : 0);
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding = strlen (this->name) + strlen (STRIPE_PATHINFO_HEADER)
+                + strlen (stripe_size_str) + 7;
+        local->xattr_total_len += (padding + 2);
+
+        pathinfo_serz = GF_CALLOC (local->xattr_total_len, sizeof (char),
+                                   gf_common_mt_char);
+        if (!pathinfo_serz)
+                goto out;
+
+        /* xlator info */
+        (void) sprintf (pathinfo_serz, "(<"STRIPE_PATHINFO_HEADER"%s:[%s]> ",
+                        this->name, stripe_size_str);
+
+        ret = stripe_xattr_aggregate (pathinfo_serz + padding, local, &tlen);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot aggregate pathinfo list");
+                goto out;
+        }
+
+        *(pathinfo_serz + padding + tlen) = ')';
+        *(pathinfo_serz + padding + tlen + 1) = '\0';
+
+        *xattr_serz = pathinfo_serz;
+
+        ret = 0;
+ out:
+        return ret;
+}
+
+/**
+ * stripe_get_matching_bs - Get the matching block size for the given path.
+ */
+int32_t
+stripe_get_matching_bs (const char *path, stripe_private_t *priv)
+{
+        struct stripe_options *trav       = NULL;
+        uint64_t               block_size = 0;
+
+        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
+        GF_VALIDATE_OR_GOTO ("stripe", path, out);
+
+        LOCK (&priv->lock);
+        {
+                block_size = priv->block_size;
+                trav = priv->pattern;
+                while (trav) {
+                        if (!fnmatch (trav->path_pattern, path, FNM_NOESCAPE)) {
+                                block_size = trav->block_size;
+                                break;
+                        }
+                        trav = trav->next;
+                }
+        }
+        UNLOCK (&priv->lock);
+
+out:
+        return block_size;
+}
+
+int32_t
+stripe_ctx_handle (xlator_t *this, call_frame_t *prev, stripe_local_t *local,
+                   dict_t *dict)
+{
+        char            key[256]       = {0,};
+        data_t         *data            = NULL;
+        int32_t         index           = 0;
+        stripe_private_t *priv          = NULL;
+
+        priv = this->private;
+
+
+        if (!local->fctx) {
+                local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
+                                         gf_stripe_mt_stripe_fd_ctx_t);
+                if (!local->fctx) {
+                        local->op_errno = ENOMEM;
+                        local->op_ret = -1;
+                        goto out;
+                }
+
+                local->fctx->static_array = 0;
+        }
+        /* Stripe block size */
+        sprintf (key, "trusted.%s.stripe-size", this->name);
+        data = dict_get (dict, key);
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-size");
+                goto out;
+        } else {
+                if (!local->fctx->stripe_size) {
+                        local->fctx->stripe_size =
+                                     data_to_int64 (data);
+                }
+
+                if (local->fctx->stripe_size != data_to_int64 (data)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "stripe-size mismatch in blocks");
+                        local->xattr_self_heal_needed = 1;
+                }
+        }
+
+        /* Stripe count */
+        sprintf (key, "trusted.%s.stripe-count", this->name);
+        data = dict_get (dict, key);
+
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-count");
+                goto out;
+        }
+        if (!local->fctx->xl_array) {
+                local->fctx->stripe_count = data_to_int32 (data);
+                if (!local->fctx->stripe_count) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error with stripe-count xattr");
+                        local->op_ret   = -1;
+                        local->op_errno = EIO;
+                        goto out;
+                }
+
+                local->fctx->xl_array = GF_CALLOC (local->fctx->stripe_count,
+                                                   sizeof (xlator_t *),
+                                                   gf_stripe_mt_xlator_t);
+
+                if (!local->fctx->xl_array) {
+                        local->op_errno = ENOMEM;
+                        local->op_ret   = -1;
+                        goto out;
+                }
+        }
+        if (local->fctx->stripe_count != data_to_int32 (data)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error with stripe-count xattr (%d != %d)",
+                        local->fctx->stripe_count, data_to_int32 (data));
+                local->op_ret   = -1;
+                local->op_errno = EIO;
+                goto out;
+        }
+
+        /* index */
+        sprintf (key, "trusted.%s.stripe-index", this->name);
+        data = dict_get (dict, key);
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-index");
+                goto out;
+        }
+        index = data_to_int32 (data);
+        if (index > priv->child_count) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error with stripe-index xattr (%d)", index);
+                local->op_ret   = -1;
+                local->op_errno = EIO;
+                goto out;
+        }
+        if (local->fctx->xl_array) {
+                if (!local->fctx->xl_array[index])
+                        local->fctx->xl_array[index] = prev->this;
+        }
+
+	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
+	data = dict_get(dict, key);
+	if (!data) {
+		/*
+		 * The file was probably created prior to coalesce support.
+		 * Assume non-coalesce mode for this file to maintain backwards
+		 * compatibility.
+		 */
+		gf_log(this->name, GF_LOG_DEBUG, "missing stripe-coalesce "
+			"attr, assume non-coalesce mode");
+		local->fctx->stripe_coalesce = 0;
+	} else {
+		local->fctx->stripe_coalesce = data_to_int32(data);
+	}
+
+
+out:
+        return 0;
+}
+
+int32_t
+stripe_xattr_request_build (xlator_t *this, dict_t *dict, uint64_t stripe_size,
+                            uint32_t stripe_count, uint32_t stripe_index,
+			    uint32_t stripe_coalesce)
+{
+        char            key[256]       = {0,};
+        int32_t         ret             = -1;
+
+        sprintf (key, "trusted.%s.stripe-size", this->name);
+        ret = dict_set_int64 (dict, key, stripe_size);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+        sprintf (key, "trusted.%s.stripe-count", this->name);
+        ret = dict_set_int32 (dict, key, stripe_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+        sprintf (key, "trusted.%s.stripe-index", this->name);
+        ret = dict_set_int32 (dict, key, stripe_index);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
+	ret = dict_set_int32(dict, key, stripe_coalesce);
+	if (ret) {
+		gf_log(this->name, GF_LOG_WARNING,
+			"failed to set %s in xattr_req_dict", key);
+		goto out;
+	}
+out:
+        return ret;
+}
+
+
+static int
+set_default_block_size (stripe_private_t *priv, char *num)
+{
+
+        int             ret = -1;
+        GF_VALIDATE_OR_GOTO ("stripe", THIS, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, num, out);
+
+
+        if (gf_string2bytesize (num, &priv->block_size) != 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "invalid number format \"%s\"", num);
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+
+}
+
+
+int
+set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
+{
+        int                    ret = -1;
+        char                  *tmp_str = NULL;
+        char                  *tmp_str1 = NULL;
+        char                  *dup_str = NULL;
+        char                  *stripe_str = NULL;
+        char                  *pattern = NULL;
+        char                  *num = NULL;
+        struct stripe_options *temp_stripeopt = NULL;
+        struct stripe_options *stripe_opt = NULL;
+
+        if (!this || !priv || !data)
+                goto out;
+
+        /* Get the pattern for striping.
+           "option block-size *avi:10MB" etc */
+        stripe_str = strtok_r (data, ",", &tmp_str);
+        while (stripe_str) {
+                dup_str = gf_strdup (stripe_str);
+                stripe_opt = GF_CALLOC (1, sizeof (struct stripe_options),
+                                        gf_stripe_mt_stripe_options);
+                if (!stripe_opt) {
+                        goto out;
+                }
+
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                num = strtok_r (NULL, ":", &tmp_str1);
+                if (!num) {
+                        num = pattern;
+                        pattern = "*";
+                        ret = set_default_block_size (priv, num);
+                        if (ret)
+                                goto out;
+                }
+                if (gf_string2bytesize (num, &stripe_opt->block_size) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format \"%s\"", num);
+                        goto out;
+                }
+
+                if (stripe_opt->block_size < STRIPE_MIN_BLOCK_SIZE) {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid Block-size: "
+                                "%s. Should be atleast %llu bytes", num,
+                                STRIPE_MIN_BLOCK_SIZE);
+                        goto out;
+                }
+                if (stripe_opt->block_size % 512) {
+                        gf_log (this->name, GF_LOG_ERROR, "Block-size: %s should"
+                                " be a multiple of 512 bytes", num);
+                        goto out;
+                }
+
+                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "block-size : pattern %s : size %"PRId64,
+                        stripe_opt->path_pattern, stripe_opt->block_size);
+
+                if (priv->pattern)
+                        temp_stripeopt = NULL;
+                else
+                        temp_stripeopt = priv->pattern;
+
+                stripe_opt->next = temp_stripeopt;
+
+                priv->pattern = stripe_opt;
+                stripe_opt = NULL;
+
+                GF_FREE (dup_str);
+                dup_str = NULL;
+
+                stripe_str = strtok_r (NULL, ",", &tmp_str);
+        }
+
+        ret = 0;
+out:
+
+        GF_FREE (dup_str);
+
+        GF_FREE (stripe_opt);
+
+        return ret;
+}
+
+int32_t
+stripe_iatt_merge (struct iatt *from, struct iatt *to)
+{
+        if (to->ia_size < from->ia_size)
+                to->ia_size = from->ia_size;
+        if (to->ia_mtime < from->ia_mtime)
+                to->ia_mtime = from->ia_mtime;
+        if (to->ia_ctime < from->ia_ctime)
+                to->ia_ctime = from->ia_ctime;
+        if (to->ia_atime < from->ia_atime)
+                to->ia_atime = from->ia_atime;
+        return 0;
+}
+
+off_t
+coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count)
+{
+	size_t line_size = 0;
+	uint64_t stripe_num = 0;
+	off_t coalesced_offset = 0;
+
+	line_size = stripe_size * stripe_count;
+	stripe_num = offset / line_size;
+
+	coalesced_offset = (stripe_num * stripe_size) +
+		(offset % stripe_size);
+
+	return coalesced_offset;
+}
+
+off_t
+uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
+		int stripe_index)
+{
+        uint64_t nr_full_stripe_chunks = 0, mod = 0;
+
+        if (!size)
+                return size;
+
+	/*
+	 * Estimate the number of fully written stripes from the
+	 * local file size. Each stripe_size chunk corresponds to
+	 * a stripe.
+	 */
+        nr_full_stripe_chunks = (size / stripe_size) * stripe_count;
+        mod = size % stripe_size;
+
+        if (!mod) {
+                /*
+		 * There is no remainder, thus we could have overestimated
+		 * the size of the file in terms of chunks. Trim the number
+		 * of chunks by the following stripe members and leave it
+		 * up to those nodes to respond with a larger size (if
+		 * necessary).
+		 */
+                nr_full_stripe_chunks -= stripe_count -
+                        (stripe_index + 1);
+                size = nr_full_stripe_chunks * stripe_size;
+        } else {
+		/*
+		 * There is a remainder and thus we own the last chunk of the
+		 * file. Add the preceding stripe members of the final stripe
+		 * along with the remainder to calculate the exact size.
+		 */
+                nr_full_stripe_chunks += stripe_index;
+                size = nr_full_stripe_chunks * stripe_size + mod;
+        }
+
+        return size;
+}
+
diff --git a/xlators/cluster/stripe/src/stripe-mem-types.h b/xlators/cluster/stripe/src/stripe-mem-types.h
new file mode 100644
index 000000000..e9ac9cf46
--- /dev/null
+++ b/xlators/cluster/stripe/src/stripe-mem-types.h
@@ -0,0 +1,31 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __STRIPE_MEM_TYPES_H__
+#define __STRIPE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_stripe_mem_types_ {
+        gf_stripe_mt_iovec = gf_common_mt_end + 1,
+        gf_stripe_mt_stripe_replies,
+        gf_stripe_mt_stripe_fd_ctx_t,
+        gf_stripe_mt_char,
+        gf_stripe_mt_int8_t,
+        gf_stripe_mt_int32_t,
+        gf_stripe_mt_xlator_t,
+        gf_stripe_mt_stripe_private_t,
+        gf_stripe_mt_stripe_options,
+        gf_stripe_mt_xattr_sort_t,
+        gf_stripe_mt_end
+};
+#endif
+
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
index 83787ca2a..69b510e23 100644
--- a/xlators/cluster/stripe/src/stripe.c
+++ b/xlators/cluster/stripe/src/stripe.c
@@ -1,3286 +1,5814 @@
 /*
-  Copyright (c) 2007, 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /**
  * xlators/cluster/stripe:
- *    Stripe translator, stripes the data accross its child nodes, 
- *    as per the options given in the volfile. The striping works 
- *    fairly simple. It writes files at different offset as per 
- *    calculation. So, 'ls -l' output at the real posix level will 
- *    show file size bigger than the actual size. But when one does 
+ *    Stripe translator, stripes the data across its child nodes,
+ *    as per the options given in the volfile. The striping works
+ *    fairly simple. It writes files at different offset as per
+ *    calculation. So, 'ls -l' output at the real posix level will
+ *    show file size bigger than the actual size. But when one does
  *    'df' or 'du <file>', real size of the file on the server is shown.
  *
  * WARNING:
  *  Stripe translator can't regenerate data if a child node gets disconnected.
- *  So, no 'self-heal' for stripe. Hence the advice, use stripe only when its 
- *  very much necessary, or else, use it in combination with AFR, to have a 
- *  backup copy. 
+ *  So, no 'self-heal' for stripe. Hence the advice, use stripe only when its
+ *  very much necessary, or else, use it in combination with AFR, to have a
+ *  backup copy.
  */
+#include <fnmatch.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include "stripe.h"
+#include "libxlator.h"
+#include "byte-order.h"
+#include "statedump.h"
 
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include <fnmatch.h>
-#include <signal.h>
+struct volume_options options[];
 
-#define STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR(_loc) do {        \
-        if (!(_loc && _loc->inode)) {                              \
-               STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL); \
-               return 0;                                           \
-        }                                                          \
-} while(0)
+int32_t
+stripe_sh_chown_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *preop, struct iatt *postop, dict_t *xdata)
+{
+        int             callcnt = -1;
+        stripe_local_t *local   = NULL;
+
+        if (!this || !frame || !frame->local) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                STRIPE_STACK_DESTROY (frame);
+        }
+out:
+        return 0;
+}
 
-/**
- * struct stripe_options : This keeps the pattern and the block-size 
- *     information, which is used for striping on a file.
- */
-struct stripe_options {
-	struct stripe_options *next;
-	char path_pattern[256];
-	uint64_t block_size;
-};
+int32_t
+stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, struct iatt *preparent,
+                          struct iatt *postparent, dict_t *xdata)
+{
+        stripe_local_t *local = NULL;
+        call_frame_t    *prev = NULL;
 
-/**
- * Private structure for stripe translator 
- */
-struct stripe_private {
-	struct stripe_options *pattern;
-	xlator_t **xl_array;
-	uint64_t   block_size;
-	gf_lock_t  lock;
-	uint8_t    nodes_down;
-	int8_t     first_child_down;
-	int8_t     child_count;
-	int8_t     state[256];       /* Current state of the child node, 
-					0 for down, 1 for up */
-	gf_boolean_t  xattr_supported;  /* 0 for no, 1 for yes, default yes */
-};
+        if (!frame || !frame->local || !cookie || !this) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
 
-/**
- * Used to keep info about the replies received from fops->readv calls 
- */
-struct readv_replies {
-	struct iovec *vector;
-	int32_t       count;    //count of vector
-	int32_t       op_ret;   //op_ret of readv
-	int32_t       op_errno;
-	struct stat   stbuf;    /* 'stbuf' is also a part of reply */
-};
+        prev  = cookie;
+        local = frame->local;
 
-/**
- * Local structure to be passed with all the frames in case of STACK_WIND
- */
-struct stripe_local; /* this itself is used inside the structure; */
-
-struct stripe_local {
-	struct stripe_local *next;
-	call_frame_t        *orig_frame; 
-
-	/* Used by _cbk functions */
- 	struct stat          stbuf;
-	struct readv_replies *replies;
-	struct statvfs       statvfs_buf;
-	dir_entry_t         *entry;
-	struct xlator_stats  stats;
-
-	int8_t               revalidate;
-	int8_t               failed;
-	int8_t               unwind;
-
-	int32_t              node_index;
-	int32_t              call_count;
-	int32_t              wind_count; /* used instead of child_cound 
-					    in case of read and write */
-	int32_t              op_ret;
-	int32_t              op_errno; 
-	int32_t              count;
-	int32_t              flags;
-	char                *name;
-	inode_t             *inode;
-
-	loc_t                loc;
-	loc_t                loc2;
-
-	/* For File I/O fops */
-	dict_t              *dict;
-
-	/* General usage */
-	off_t                offset;
-	off_t                stripe_size;
-
-	int8_t              *list;
-	struct flock         lock;
-	fd_t                *fd;
-	void                *value;
-};
+        STACK_WIND (frame, stripe_sh_chown_cbk, prev->this,
+                    prev->this->fops->setattr, &local->loc,
+                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
 
-typedef struct stripe_local   stripe_local_t;
-typedef struct stripe_private stripe_private_t;
+out:
+        return 0;
+}
 
-/**
- * stripe_get_matching_bs - Get the matching block size for the given path.
- */
-int32_t 
-stripe_get_matching_bs (const char *path, 
-			struct stripe_options *opts,
-			uint64_t default_bs) 
-{
-	struct stripe_options *trav       = NULL;
-	char                  *pathname   = NULL;
-	uint64_t               block_size = 0;
-
-	block_size = default_bs;
-	pathname   = strdup (path);
-	trav       = opts;
-
-	while (trav) {
-		if (fnmatch (trav->path_pattern, 
-			     pathname, FNM_NOESCAPE) == 0) {
-			block_size = trav->block_size;
-			break;
-		}
-		trav = trav->next;
-	}
-	free (pathname);
-	
-	return block_size;
+int32_t
+stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
+                        stripe_local_t *local)
+{
+        xlator_list_t    *trav   = NULL;
+        call_frame_t     *rframe = NULL;
+        stripe_local_t   *rlocal = NULL;
+        stripe_private_t *priv   = NULL;
+        dict_t           *xdata  = NULL;
+        int               ret    = 0;
+
+        if (!local || !this || !frame) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        if (!(IA_ISREG (local->stbuf.ia_type) ||
+              IA_ISDIR (local->stbuf.ia_type)))
+                return 0;
+
+        priv = this->private;
+        trav = this->children;
+        rframe = copy_frame (frame);
+        if (!rframe) {
+                goto out;
+        }
+        rlocal = mem_get0 (this->local_pool);
+        if (!rlocal) {
+                goto out;
+        }
+        rframe->local = rlocal;
+        rlocal->call_count = priv->child_count;
+        loc_copy (&rlocal->loc, &local->loc);
+        memcpy (&rlocal->stbuf, &local->stbuf, sizeof (struct iatt));
+
+        xdata = dict_new ();
+        if (!xdata)
+                goto out;
+
+        ret = dict_set_static_bin (xdata, "gfid-req", local->stbuf.ia_gfid, 16);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set gfid-req", local->loc.path);
+
+        while (trav) {
+                if (IA_ISREG (local->stbuf.ia_type)) {
+                        STACK_WIND (rframe, stripe_sh_make_entry_cbk,
+                                    trav->xlator, trav->xlator->fops->mknod,
+                                    &local->loc,
+                                    st_mode_from_ia (local->stbuf.ia_prot,
+                                                     local->stbuf.ia_type),
+                                    0, 0, xdata);
+                }
+                if (IA_ISDIR (local->stbuf.ia_type)) {
+                        STACK_WIND (rframe, stripe_sh_make_entry_cbk,
+                                    trav->xlator, trav->xlator->fops->mkdir,
+                                    &local->loc,
+                                    st_mode_from_ia (local->stbuf.ia_prot,
+                                                     local->stbuf.ia_type),
+                                    0, xdata);
+                }
+                trav = trav->next;
+        }
+
+        if (xdata)
+                dict_unref (xdata);
+        return 0;
+
+out:
+        if (rframe)
+                STRIPE_STACK_DESTROY (rframe);
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
 }
 
 
-/*
- * stripe_common_cbk -
- */
 int32_t
-stripe_common_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno)
+stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+        int32_t         callcnt     = 0;
+        stripe_local_t *local       = NULL;
+        call_frame_t   *prev        = NULL;
+        int             ret         = 0;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        if (op_errno != ENOENT)
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "%s returned error %s",
+                                        prev->this->name,
+                                        strerror (op_errno));
+                        if (local->op_errno != ESTALE)
+                                local->op_errno = op_errno;
+                        if (((op_errno != ENOENT) && (op_errno != ENOTCONN)) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                        if (op_errno == ENOENT)
+                                local->entry_self_heal_needed = 1;
+                }
+
+                if (op_ret >= 0) {
+                        local->op_ret = 0;
+                        if (IA_ISREG (buf->ia_type)) {
+                                ret = stripe_ctx_handle (this, prev, local,
+                                                         xdata);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                 "Error getting fctx info from"
+                                                 " dict");
+                        }
+
+                        if (FIRST_CHILD(this) == prev->this) {
+                                local->stbuf      = *buf;
+                                local->postparent = *postparent;
+                                local->inode = inode_ref (inode);
+                                if (xdata)
+                                        local->xdata = dict_ref (xdata);
+                                if (local->xattr) {
+                                        stripe_aggregate_xattr (local->xdata,
+                                                                local->xattr);
+                                        dict_unref (local->xattr);
+                                        local->xattr = NULL;
+                                }
+                        }
+
+                        if (!local->xdata && !local->xattr) {
+                                local->xattr = dict_ref (xdata);
+                        } else if (local->xdata) {
+                                stripe_aggregate_xattr (local->xdata, xdata);
+                        } else if (local->xattr) {
+                                stripe_aggregate_xattr (local->xattr, xdata);
+                        }
+
+                        local->stbuf_blocks      += buf->ia_blocks;
+                        local->postparent_blocks += postparent->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
+                        if (local->stbuf_size < buf->ia_size)
+                                local->stbuf_size = buf->ia_size;
+                        if (local->postparent_size < postparent->ia_size)
+                                local->postparent_size = postparent->ia_size;
+
+                        if (uuid_is_null (local->ia_gfid))
+                                uuid_copy (local->ia_gfid, buf->ia_gfid);
+
+                        /* Make sure the gfid on all the nodes are same */
+                        if (uuid_compare (local->ia_gfid, buf->ia_gfid)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "%s: gfid different on subvolume %s",
+                                        local->loc.path, prev->this->name);
+                        }
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->op_ret == 0 && local->entry_self_heal_needed &&
+                    !uuid_is_null (local->loc.inode->gfid))
+                        stripe_entry_self_heal (frame, this, local);
+
+                if (local->failed)
+                        local->op_ret = -1;
+
+                if (local->op_ret != -1) {
+                        local->stbuf.ia_blocks      = local->stbuf_blocks;
+                        local->stbuf.ia_size        = local->stbuf_size;
+                        local->postparent.ia_blocks = local->postparent_blocks;
+                        local->postparent.ia_size   = local->postparent_size;
+                        inode_ctx_put (local->inode, this,
+                                       (uint64_t) (long)local->fctx);
+                }
+
+                STRIPE_STACK_UNWIND (lookup, frame, local->op_ret,
+                                     local->op_errno, local->inode,
+                                     &local->stbuf, local->xdata,
+                                     &local->postparent);
+        }
+out:
+        return 0;
 }
 
-/**
- * stripe_stack_unwind_cbk -  This function is used for all the _cbk without 
- *     any extra arguments (other than the minimum given)
- * This is called from functions like fsync,unlink,rmdir etc.
- *
- */
-int32_t 
-stripe_stack_unwind_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
+int32_t
+stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               dict_t *xdata)
 {
-	int32_t         callcnt = 0;
-	stripe_local_t *local   = frame->local;
+        stripe_local_t   *local    = NULL;
+        xlator_list_t    *trav     = NULL;
+        stripe_private_t *priv     = NULL;
+        int32_t           op_errno = EINVAL;
+        int64_t           filesize = 0;
+        int               ret      = 0;
+        uint64_t          tmpctx   = 0;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        loc_copy (&local->loc, loc);
+
+        inode_ctx_get (local->inode, this, &tmpctx);
+        if (tmpctx)
+                local->fctx = (stripe_fd_ctx_t*) (long)tmpctx;
+
+        /* quick-read friendly changes */
+        if (xdata && dict_get (xdata, GF_CONTENT_KEY)) {
+                ret = dict_get_int64 (xdata, GF_CONTENT_KEY, &filesize);
+                if (!ret && (filesize > priv->block_size))
+                        dict_del (xdata, GF_CONTENT_KEY);
+        }
+
+        /* get stripe-size xattr on lookup. This would be required for
+         * open/read/write/pathinfo calls. Hence we send down the request
+         * even when type == IA_INVAL */
+
+	/*
+	 * We aren't guaranteed to have xdata here. We need the format info for
+	 * the file, so allocate xdata if necessary.
+	 */
+	if (!xdata)
+		xdata = dict_new();
+	else
+		xdata = dict_ref(xdata);
+
+        if (xdata && (IA_ISREG (loc->inode->ia_type) ||
+            (loc->inode->ia_type == IA_INVAL))) {
+                ret = stripe_xattr_request_build (this, xdata, 8, 4, 4, 0);
+                if (ret)
+                        gf_log (this->name , GF_LOG_ERROR, "Failed to build"
+                                " xattr request for %s", loc->path);
+
+        }
+
+        /* Everytime in stripe lookup, all child nodes
+           should be looked up */
+        local->call_count = priv->child_count;
+        while (trav) {
+                STACK_WIND (frame, stripe_lookup_cbk, trav->xlator,
+                            trav->xlator->fops->lookup, loc, xdata);
+                trav = trav->next;
+        }
+
+	dict_unref(xdata);
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        return 0;
+}
 
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));			
-			local->op_errno = op_errno;
-			if (op_errno == ENOTCONN) 
-				local->failed = 1;
-		}
-		if (op_ret >= 0) 
-			local->op_ret = op_ret;
+
+int32_t
+stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+
+                if (op_ret == 0) {
+                        local->op_ret = 0;
+
+                        if (FIRST_CHILD(this) == prev->this) {
+                                local->stbuf = *buf;
+                        }
+
+                        local->stbuf_blocks += buf->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
+                        if (local->stbuf_size < buf->ia_size)
+                                local->stbuf_size = buf->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                if (local->op_ret != -1) {
+                        local->stbuf.ia_size   = local->stbuf_size;
+                        local->stbuf.ia_blocks = local->stbuf_blocks;
+                }
+
+                STRIPE_STACK_UNWIND (stat, frame, local->op_ret,
+                                     local->op_errno, &local->stbuf, NULL);
+        }
+out:
+        return 0;
+}
+
+int32_t
+stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        xlator_list_t    *trav = NULL;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+	if (IA_ISREG(loc->inode->ia_type)) {
+		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
 	}
-	UNLOCK (&frame->lock);
 
-	if (!callcnt) {
-		if (local->failed)
-			local->op_ret = -1;
+        while (trav) {
+                STACK_WIND (frame, stripe_stat_cbk, trav->xlator,
+                            trav->xlator->fops->stat, loc, NULL);
+                trav = trav->next;
+        }
 
-		if (local->loc.path)
-			loc_wipe (&local->loc);
-		if (local->loc2.path)
-			loc_wipe (&local->loc2);
+        return 0;
 
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-	return 0;
+err:
+        STRIPE_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+        return 0;
 }
 
-int32_t 
-stripe_common_buf_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct stat *buf)
+
+int32_t
+stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
+        stripe_local_t *local = NULL;
+        int32_t         callcnt = 0;
+
+        if (!this || !frame || !frame->local) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+        local = frame->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret && (op_errno != ENOTCONN)) {
+                        local->op_errno = op_errno;
+                }
+                if (op_ret == 0) {
+                        struct statvfs *dict_buf = &local->statvfs_buf;
+                        dict_buf->f_bsize   = stbuf->f_bsize;
+                        dict_buf->f_frsize  = stbuf->f_frsize;
+                        dict_buf->f_blocks += stbuf->f_blocks;
+                        dict_buf->f_bfree  += stbuf->f_bfree;
+                        dict_buf->f_bavail += stbuf->f_bavail;
+                        dict_buf->f_files  += stbuf->f_files;
+                        dict_buf->f_ffree  += stbuf->f_ffree;
+                        dict_buf->f_favail += stbuf->f_favail;
+                        dict_buf->f_fsid    = stbuf->f_fsid;
+                        dict_buf->f_flag    = stbuf->f_flag;
+                        dict_buf->f_namemax = stbuf->f_namemax;
+                        local->op_ret = 0;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                STRIPE_STACK_UNWIND (statfs, frame, local->op_ret,
+                                     local->op_errno, &local->statvfs_buf, NULL);
+        }
+out:
+        return 0;
 }
 
-/**
- * stripe_stack_unwind_buf_cbk -  This function is used for all the _cbk with 
- *    'struct stat *buf' as extra argument (other than minimum)
- * This is called from functions like, chmod, fchmod, chown, fchown,
- * truncate, ftruncate, utimens etc.
- *
- * @cookie - this argument should be always 'xlator_t *' of child node 
- */
-int32_t 
-stripe_stack_unwind_buf_cbk (call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct stat *buf)
+int32_t
+stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-			if (op_errno == ENOTCONN)
-				local->failed = 1;
-		}
-    
-		if (op_ret == 0) {
-			local->op_ret = 0;
-			if (local->stbuf.st_blksize == 0) {
-				local->stbuf = *buf;
-				/* Because st_blocks gets added again */
-				local->stbuf.st_blocks = 0;
-			}
-
-			if (FIRST_CHILD(this) == 
-			    ((call_frame_t *)cookie)->this) {
-				/* Always, pass the inode number of 
-				   first child to the above layer */
-				local->stbuf.st_ino = buf->st_ino;
-				local->stbuf.st_mtime = buf->st_mtime;
-			}
-
-			local->stbuf.st_blocks += buf->st_blocks;
-			if (local->stbuf.st_size < buf->st_size)
-				local->stbuf.st_size = buf->st_size;
-			if (local->stbuf.st_blksize != buf->st_blksize) {
-				/* TODO: add to blocks in terms of 
-				   original block size */
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		if (local->failed)
-			local->op_ret = -1;
+        stripe_local_t   *local = NULL;
+        xlator_list_t    *trav = NULL;
+        stripe_private_t *priv = NULL;
+        int32_t           op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        trav = this->children;
+        priv = this->private;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        frame->local = local;
+
+        local->call_count = priv->child_count;
+        while (trav) {
+                STACK_WIND (frame, stripe_statfs_cbk, trav->xlator,
+                            trav->xlator->fops->statfs, loc, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
 
-		if (local->loc.path)
-			loc_wipe (&local->loc);
-		if (local->loc2.path)
-			loc_wipe (&local->loc2);
 
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
 
-	return 0;
+int32_t
+stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+
+                if (op_ret == 0) {
+                        local->op_ret = 0;
+                        if (FIRST_CHILD(this) == prev->this) {
+                                local->pre_buf  = *prebuf;
+                                local->post_buf = *postbuf;
+                        }
+
+                        local->prebuf_blocks  += prebuf->ia_blocks;
+                        local->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, local->fctx, prev);
+			correct_file_size(postbuf, local->fctx, prev);
+
+                        if (local->prebuf_size < prebuf->ia_size)
+                                local->prebuf_size = prebuf->ia_size;
+
+                        if (local->postbuf_size < postbuf->ia_size)
+                                local->postbuf_size = postbuf->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                if (local->op_ret != -1) {
+                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
+                        local->pre_buf.ia_size    = local->prebuf_size;
+                        local->post_buf.ia_blocks = local->postbuf_blocks;
+                        local->post_buf.ia_size   = local->postbuf_size;
+                }
+
+                STRIPE_STACK_UNWIND (truncate, frame, local->op_ret,
+                                     local->op_errno, &local->pre_buf,
+                                     &local->post_buf, NULL);
+        }
+out:
+        return 0;
 }
 
-/* In case of symlink, mknod, the file is created on just first node */
-int32_t 
-stripe_common_inode_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 inode_t *inode,
-			 struct stat *buf)
+int32_t
+stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = EINVAL;
+	int		  i, eof_idx;
+	off_t		  dest_offset, tmp_offset;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+	inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
+		op_errno = EINVAL;
+		goto err;
+	}
 
-/**
- * stripe_stack_unwind_inode_cbk - This is called by the function like, 
- *                   link (), symlink (), mkdir (), mknod () 
- *           This creates a inode for new inode. It keeps a list of all 
- *           the inodes received from the child nodes. It is used while 
- *           forwarding any fops to child nodes.
- *
- */
-int32_t 
-stripe_stack_unwind_inode_cbk (call_frame_t *frame,
-			       void *cookie,
-			       xlator_t *this,
-			       int32_t op_ret,
-			       int32_t op_errno,
-			       inode_t *inode,
-			       struct stat *buf)
-{
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
+	local->fctx = fctx;
+	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
 
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-			if (op_errno == ENOTCONN)
-				local->failed = 1;
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (!fctx->xl_array[i]) {
+			gf_log(this->name, GF_LOG_ERROR,
+				"no xlator at index %d", i);
+			op_errno = EINVAL;
+			goto err;
 		}
- 
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (local->stbuf.st_blksize == 0) {
-				local->inode = inode;
-				local->stbuf = *buf;
-				/* Because st_blocks gets added again */
-				local->stbuf.st_blocks = 0;
-			}
-			if (FIRST_CHILD(this) == 
-			    ((call_frame_t *)cookie)->this) {
-				local->stbuf.st_ino = buf->st_ino;
-				local->stbuf.st_mtime = buf->st_mtime;
-			}
 
-			local->stbuf.st_blocks += buf->st_blocks;
-			if (local->stbuf.st_size < buf->st_size)
-				local->stbuf.st_size = buf->st_size;
-			if (local->stbuf.st_blksize != buf->st_blksize) {
-				/* TODO: add to blocks in terms of 
-				   original block size */
-			}
+		if (fctx->stripe_coalesce) {
+			/*
+	 		 * The node that owns EOF is truncated to the exact
+	 		 * coalesced offset. Nodes prior to this index should
+	 		 * be rounded up to the size of the complete stripe,
+	 		 * while nodes after this index should be rounded down
+			 * to the size of the previous stripe.
+			 */
+			if (i < eof_idx)
+				tmp_offset = roof(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else if (i > eof_idx)
+				tmp_offset = floor(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else
+				tmp_offset = offset;
+
+			dest_offset = coalesced_offset(tmp_offset,
+					fctx->stripe_size, fctx->stripe_count);
+		} else {
+			dest_offset = offset;
 		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		if (local->failed)
-			local->op_ret = -1;
 
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local->inode, &local->stbuf);
+		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
+			fctx->xl_array[i]->fops->truncate, loc, dest_offset,
+			NULL);
 	}
 
-	return 0;
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
 }
 
-int32_t 
-stripe_stack_unwind_inode_lookup_cbk (call_frame_t *frame,
-				      void *cookie,
-				      xlator_t *this,
-				      int32_t op_ret,
-				      int32_t op_errno,
-				      inode_t *inode,
-				      struct stat *buf,
-				      dict_t *dict)
-{
-	int32_t callcnt = 0;
-	dict_t *tmp_dict = NULL;
-	stripe_local_t *local = frame->local;
 
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			if (op_errno != ENOENT)
-				gf_log (this->name, GF_LOG_WARNING, 
-					"%s returned error %s",
-					((call_frame_t *)cookie)->this->name,
-					strerror (op_errno));
-			local->op_errno = op_errno;
-			if (op_errno == ENOTCONN)
-				local->failed = 1;
-		}
- 
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (local->stbuf.st_blksize == 0) {
-				local->inode = inode;
-				local->stbuf = *buf;
-				/* Because st_blocks gets added again */
-				local->stbuf.st_blocks = 0;
-			}
-			if (FIRST_CHILD(this) == 
-			    ((call_frame_t *)cookie)->this) {
-				local->stbuf.st_ino = buf->st_ino;
-				local->stbuf.st_mtime = buf->st_mtime;
-				if (local->dict)
-					dict_unref (local->dict);
-				local->dict = dict_ref (dict);
-			} else {
-				if (!local->dict)
-					local->dict = dict_ref (dict);
-			}
-			local->stbuf.st_blocks += buf->st_blocks;
-			if (local->stbuf.st_size < buf->st_size)
-				local->stbuf.st_size = buf->st_size;
-			if (local->stbuf.st_blksize != buf->st_blksize) {
-				/* TODO: add to blocks in terms of 
-				   original block size */
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
+int32_t
+stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *preop, struct iatt *postop, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+
+                if (op_ret == 0) {
+                        local->op_ret = 0;
+
+                        if (FIRST_CHILD(this) == prev->this) {
+                                local->pre_buf  = *preop;
+                                local->post_buf = *postop;
+                        }
+
+                        local->prebuf_blocks  += preop->ia_blocks;
+                        local->postbuf_blocks += postop->ia_blocks;
+
+			correct_file_size(preop, local->fctx, prev);
+			correct_file_size(postop, local->fctx, prev);
+
+                        if (local->prebuf_size < preop->ia_size)
+                                local->prebuf_size = preop->ia_size;
+                        if (local->postbuf_size < postop->ia_size)
+                                local->postbuf_size = postop->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                if (local->op_ret != -1) {
+                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
+                        local->pre_buf.ia_size    = local->prebuf_size;
+                        local->post_buf.ia_blocks = local->postbuf_blocks;
+                        local->post_buf.ia_size   = local->postbuf_size;
+                }
+
+                STRIPE_STACK_UNWIND (setattr, frame, local->op_ret,
+                                     local->op_errno, &local->pre_buf,
+                                     &local->post_buf, NULL);
+        }
+out:
+        return 0;
+}
 
-	if (!callcnt) {
-		if (local->failed)
-			local->op_ret = -1;
 
-		tmp_dict = local->dict;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local->inode, &local->stbuf, local->dict);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
+int32_t
+stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        xlator_list_t    *trav = NULL;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t	 *fctx = NULL;
+        int32_t           op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        if (!IA_ISDIR (loc->inode->ia_type) &&
+            !IA_ISREG (loc->inode->ia_type)) {
+                local->call_count = 1;
+                STACK_WIND (frame, stripe_setattr_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->setattr,
+                            loc, stbuf, valid, NULL);
+                return 0;
+        }
+
+	if (IA_ISREG(loc->inode->ia_type)) {
+		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
 	}
 
-	return 0;
+        local->call_count = priv->child_count;
+        while (trav) {
+                STACK_WIND (frame, stripe_setattr_cbk,
+                            trav->xlator, trav->xlator->fops->setattr,
+                            loc, stbuf, valid, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
 }
 
 
-/**
- * stripe_lookup -
- */
-int32_t 
-stripe_lookup (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       dict_t *xattr_req)
-{
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = NULL;
-	stripe_private_t *priv = this->private;
-	char send_lookup_to_all = 0;
-
-	if (!(loc && loc->inode)) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"wrong argument, returning EINVAL");
-		STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-
-	if ((!loc->inode->st_mode) || 
-	    S_ISDIR (loc->inode->st_mode) || 
-	    S_ISREG (loc->inode->st_mode))
-		send_lookup_to_all = 1;
-
-	if (send_lookup_to_all) {
-		/* Everytime in stripe lookup, all child nodes 
-		   should be looked up */
-		local->call_count = priv->child_count;
-		trav = this->children;
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_stack_unwind_inode_lookup_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->lookup,
-				    loc, xattr_req);
-			trav = trav->next;
-		}
-	} else {
-		local->call_count = 1;
-		
-		STACK_WIND (frame,
-			    stripe_stack_unwind_inode_lookup_cbk,
-			    FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->lookup,
-			    loc, xattr_req);
-	}
-  
-	return 0;
+int32_t
+stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        xlator_list_t    *trav = NULL;
+        int32_t           op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_setattr_cbk, trav->xlator,
+                            trav->xlator->fops->fsetattr, fd, stbuf, valid, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
 }
 
-/**
- * stripe_stat -
- */
 int32_t
-stripe_stat (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{ 
-	int send_lookup_to_all = 0;
-	xlator_list_t *trav = NULL;
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	if (S_ISDIR (loc->inode->st_mode) || S_ISREG (loc->inode->st_mode))
-		send_lookup_to_all = 1;
-
-	if (!send_lookup_to_all) {
-		STACK_WIND (frame,
-			    stripe_common_buf_cbk,
-			    FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->stat,
-			    loc);
-	} else {
-		/* Initialization */
-		local = CALLOC (1, sizeof (stripe_local_t));
-		ERR_ABORT (local);
-		local->op_ret = -1;
-		frame->local = local;
-		local->inode = loc->inode;
-		local->call_count = priv->child_count;
-    
-		trav = this->children;
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_stack_unwind_buf_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->stat,
-				    loc);
-			trav = trav->next;
-		}
-	}
-	return 0;
+stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         struct iatt *preoldparent, struct iatt *postoldparent,
+                         struct iatt *prenewparent, struct iatt *postnewparent,
+                         dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+
+                if (op_ret == 0) {
+                        local->op_ret = 0;
+
+                        local->stbuf.ia_blocks      += buf->ia_blocks;
+                        local->preparent.ia_blocks  += preoldparent->ia_blocks;
+                        local->postparent.ia_blocks += postoldparent->ia_blocks;
+                        local->pre_buf.ia_blocks    += prenewparent->ia_blocks;
+                        local->post_buf.ia_blocks   += postnewparent->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
+                        if (local->stbuf.ia_size < buf->ia_size)
+                                local->stbuf.ia_size =  buf->ia_size;
+
+                        if (local->preparent.ia_size < preoldparent->ia_size)
+                                local->preparent.ia_size = preoldparent->ia_size;
+
+                        if (local->postparent.ia_size < postoldparent->ia_size)
+                                local->postparent.ia_size = postoldparent->ia_size;
+
+                        if (local->pre_buf.ia_size < prenewparent->ia_size)
+                                local->pre_buf.ia_size = prenewparent->ia_size;
+
+                        if (local->post_buf.ia_size < postnewparent->ia_size)
+                                local->post_buf.ia_size = postnewparent->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                STRIPE_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                                     &local->stbuf, &local->preparent,
+                                     &local->postparent,  &local->pre_buf,
+                                     &local->post_buf, NULL);
+        }
+out:
+        return 0;
 }
 
-
-/**
- * stripe_chmod -
- */
 int32_t
-stripe_chmod (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      mode_t mode)
+stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         struct iatt *preoldparent, struct iatt *postoldparent,
+                         struct iatt *prenewparent, struct iatt *postnewparent,
+                         dict_t *xdata)
 {
-	int send_fop_to_all = 0;
-	xlator_list_t *trav = NULL;
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
+        stripe_local_t *local = NULL;
+        xlator_list_t  *trav = NULL;
+
+        if (!this || !frame || !frame->local) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                op_errno = EINVAL;
+                goto unwind;
+        }
+
+        if (op_ret == -1) {
+                goto unwind;
+        }
+
+        local = frame->local;
+        trav = this->children;
+
+        local->stbuf      = *buf;
+        local->preparent  = *preoldparent;
+        local->postparent = *postoldparent;
+        local->pre_buf    = *prenewparent;
+        local->post_buf   = *postnewparent;
+
+        local->op_ret = 0;
+        local->call_count--;
+
+        trav = trav->next; /* Skip first child */
+        while (trav) {
+                STACK_WIND (frame, stripe_stack_rename_cbk,
+                            trav->xlator, trav->xlator->fops->rename,
+                            &local->loc, &local->loc2, NULL);
+                trav = trav->next;
+        }
+        return 0;
+
+unwind:
+        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, buf, preoldparent,
+                             postoldparent, prenewparent, postnewparent, NULL);
+        return 0;
+}
 
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning ENOTCONN");
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
+int32_t
+stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+        stripe_private_t *priv = NULL;
+        stripe_local_t   *local = NULL;
+        xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t	 *fctx = NULL;
+        int32_t           op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (oldloc, err);
+        VALIDATE_OR_GOTO (oldloc->path, err);
+        VALIDATE_OR_GOTO (oldloc->inode, err);
+        VALIDATE_OR_GOTO (newloc, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* If any one node is down, don't allow rename */
+        if (priv->nodes_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        loc_copy (&local->loc, oldloc);
+        loc_copy (&local->loc2, newloc);
+
+        local->call_count = priv->child_count;
+
+	if (IA_ISREG(oldloc->inode->ia_type)) {
+		inode_ctx_get(oldloc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
 	}
 
-	if (S_ISDIR (loc->inode->st_mode) || S_ISREG (loc->inode->st_mode))
-		send_fop_to_all = 1;
-
-	if (!send_fop_to_all) {
-		STACK_WIND (frame,
-			    stripe_common_buf_cbk,
-			    FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->chmod,
-			    loc, mode);
-	} else {
-		/* Initialization */
-		local = CALLOC (1, sizeof (stripe_local_t));
-		ERR_ABORT (local);
-		local->op_ret = -1;
-		frame->local = local;
-		local->inode = loc->inode;
-		local->call_count = priv->child_count;
-
-		trav = this->children;
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_stack_unwind_buf_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->chmod,
-				    loc, mode);
-			trav = trav->next;
-		}
-	}
-	return 0;
-}
+        frame->local = local;
 
+        STACK_WIND (frame, stripe_first_rename_cbk, trav->xlator,
+                    trav->xlator->fops->rename, oldloc, newloc, NULL);
 
-/**
- * stripe_chown - 
- */
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL, NULL);
+        return 0;
+}
 int32_t
-stripe_chown (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      uid_t uid,
-	      gid_t gid)
+stripe_first_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
-	int send_fop_to_all = 0;
-	xlator_list_t *trav = NULL;
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning ENOTCONN");
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
-	}
+        stripe_local_t *local   = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+        local->op_ret = 0;
+        local->preparent  = *preparent;
+        local->postparent = *postparent;
+        local->preparent_blocks  += preparent->ia_blocks;
+        local->postparent_blocks += postparent->ia_blocks;
+
+        STRIPE_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                            &local->preparent, &local->postparent, xdata);
+        return 0;
+out:
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
 
-	if (S_ISDIR (loc->inode->st_mode) || S_ISREG (loc->inode->st_mode))
-		send_fop_to_all = 1;
-
-	trav = this->children;
-	if (!send_fop_to_all) {
-		STACK_WIND (frame,
-			    stripe_common_buf_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->chown,
-			    loc, uid, gid);
-	} else {
-		/* Initialization */
-		local = CALLOC (1, sizeof (stripe_local_t));
-		ERR_ABORT (local);
-		local->op_ret = -1;
-		frame->local = local;
-		local->inode = loc->inode;
-		local->call_count = priv->child_count;
-
-		trav = this->children;
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_stack_unwind_buf_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->chown,
-				    loc, uid, gid);
-			trav = trav->next;
-		}
-	}
 
-	return 0;
-}
 
 
-/**
- * stripe_statfs_cbk - 
- */
 int32_t
-stripe_statfs_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct statvfs *stbuf)
-{
-	stripe_local_t *local = (stripe_local_t *)frame->local;
-	int32_t callcnt;
-	LOCK(&frame->lock);
-	{
-		callcnt = --local->call_count;
-
-		if (op_ret != 0 && op_errno != ENOTCONN) {
-			local->op_errno = op_errno;
-		}
-		if (op_ret == 0) {
-      			struct statvfs *dict_buf = &local->statvfs_buf;
-			dict_buf->f_bsize   = stbuf->f_bsize;
-			dict_buf->f_frsize  = stbuf->f_frsize;
-			dict_buf->f_blocks += stbuf->f_blocks;
-			dict_buf->f_bfree  += stbuf->f_bfree;
-			dict_buf->f_bavail += stbuf->f_bavail;
-			dict_buf->f_files  += stbuf->f_files;
-			dict_buf->f_ffree  += stbuf->f_ffree;
-			dict_buf->f_favail += stbuf->f_favail;
-			dict_buf->f_fsid    = stbuf->f_fsid;
-			dict_buf->f_flag    = stbuf->f_flag;
-			dict_buf->f_namemax = stbuf->f_namemax;
-			local->op_ret = 0;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, &local->statvfs_buf);
-	}
-  
-	return 0;
+stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local   = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if (op_errno != ENOENT) {
+                                local->failed = 1;
+                                local->op_ret = op_ret;
+                        }
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (callcnt == 1) {
+                if (local->failed) {
+                        op_errno = local->op_errno;
+                        goto out;
+                }
+                STACK_WIND(frame, stripe_first_unlink_cbk, FIRST_CHILD (this),
+                           FIRST_CHILD (this)->fops->unlink, &local->loc,
+                           local->xflag, local->xdata);
+        }
+        return 0;
+out:
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
 }
 
-
-/**
- * stripe_statfs - 
- */
 int32_t
-stripe_statfs (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc)
-{
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	frame->local = local;
-
-	local->call_count = ((stripe_private_t *)this->private)->child_count;
-	while (trav) {
-		STACK_WIND (frame,
-			    stripe_statfs_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->statfs,
-			    loc);
-		trav = trav->next;
-	}
-
-	return 0;
+stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int xflag, dict_t *xdata)
+{
+        xlator_list_t    *trav = NULL;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        int32_t           op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Don't unlink a file if a node is down */
+        if (priv->nodes_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        loc_copy (&local->loc, loc);
+        local->xflag = xflag;
+
+        if (xdata)
+            local->xdata = dict_ref (xdata);
+
+        frame->local = local;
+        local->call_count = priv->child_count;
+        trav = trav->next; /* Skip the first child */
+
+        while (trav) {
+                STACK_WIND (frame, stripe_unlink_cbk,
+                            trav->xlator, trav->xlator->fops->unlink,
+                            loc, xflag, xdata);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
 }
 
 
-/**
- * stripe_truncate - 
- */
 int32_t
-stripe_truncate (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 off_t offset)
+stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-	int send_fop_to_all = 0;
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
+        stripe_local_t *local = NULL;
 
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
+        if (!this || !frame || !frame->local) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning ENOTCONN");
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
-	}
+        if (op_ret == -1) {
+                goto err;
+        }
 
-	if (S_ISDIR (loc->inode->st_mode) || S_ISREG (loc->inode->st_mode))
-		send_fop_to_all = 1;
-
-	if (!send_fop_to_all) {
-		STACK_WIND (frame,
-			    stripe_common_buf_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->truncate,
-			    loc,
-			    offset);
-	} else {
-		/* Initialization */
-		local = CALLOC (1, sizeof (stripe_local_t));
-		ERR_ABORT (local);
-		local->op_ret = -1;
-		frame->local = local;
-		local->inode = loc->inode;
-		local->call_count = priv->child_count;
-    
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_stack_unwind_buf_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->truncate,
-				    loc,
-				    offset);
-			trav = trav->next;
-		}
-	}
+        local = frame->local;
+        local->op_ret = 0;
 
-	return 0;
-}
+        local->call_count--; /* First child successful */
 
+        local->preparent  = *preparent;
+        local->postparent = *postparent;
+        local->preparent_size  = preparent->ia_size;
+        local->postparent_size = postparent->ia_size;
+        local->preparent_blocks  += preparent->ia_blocks;
+        local->postparent_blocks += postparent->ia_blocks;
 
-/**
- * stripe_utimens - 
- */
-int32_t 
-stripe_utimens (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		struct timespec tv[2])
-{
-	int send_fop_to_all = 0;
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning ENOTCONN");
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
-	}
+        STRIPE_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, xdata);
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (rmdir, frame, op_ret, op_errno, NULL, NULL, NULL);
+        return 0;
 
-	if (S_ISDIR (loc->inode->st_mode) || S_ISREG (loc->inode->st_mode))
-		send_fop_to_all = 1;
-
-	if (!send_fop_to_all) {
-		STACK_WIND (frame,
-			    stripe_common_buf_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->utimens,
-			    loc, tv);
-	} else {
-		/* Initialization */
-		local = CALLOC (1, sizeof (stripe_local_t));
-		ERR_ABORT (local);
-		local->op_ret = -1;
-		frame->local = local;
-		local->inode = loc->inode;
-		local->call_count = priv->child_count;
-    
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_stack_unwind_buf_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->utimens,
-				    loc, tv);
-			trav = trav->next;
-		}
-	}
-	return 0;
 }
 
-
-int32_t 
-stripe_first_rename_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 struct stat *buf)
+int32_t
+stripe_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	stripe_local_t *local = frame->local;
-	xlator_list_t *trav = this->children;
+        int32_t         callcnt = 0;
+        stripe_local_t *local   = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
+                                prev->this->name, strerror (op_errno));
+                        if (op_errno != ENOENT)
+                                local->failed = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (callcnt == 1) {
+                if (local->failed)
+                        goto out;
+                STACK_WIND (frame, stripe_first_rmdir_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->rmdir, &local->loc,
+                            local->flags, NULL);
+        }
+        return 0;
+out:
+        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
-	if (op_ret == -1) 
-	{
-		STACK_UNWIND (frame, op_ret, op_errno, buf);
-		return 0;
-	}
+int32_t
+stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
+{
+        xlator_list_t    *trav = NULL;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        int32_t           op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* don't delete a directory if any of the subvolume is down */
+        if (priv->nodes_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        loc_copy (&local->loc, loc);
+        local->flags = flags;
+        local->call_count = priv->child_count;
+        trav = trav->next; /* skip the first child */
+
+        while (trav) {
+                STACK_WIND (frame, stripe_rmdir_cbk,  trav->xlator,
+                            trav->xlator->fops->rmdir, loc, flags, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
-	local->op_ret = 0;
-	local->stbuf = *buf;
-	local->call_count--;
-	trav = trav->next; /* Skip first child */
-
-	while (trav) {
-		STACK_WIND (frame,
-			    stripe_stack_unwind_buf_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->rename,
-			    &local->loc, &local->loc2);
-		trav = trav->next;
-	}
 
-	return 0;
+int32_t
+stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int32_t op_ret,
+                                    int32_t op_errno, struct iatt *preparent,
+                                    struct iatt *postparent, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+
+        if (!this || !frame || !frame->local) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
+                                     local->inode, &local->stbuf,
+                                     &local->preparent, &local->postparent, NULL);
+        }
+out:
+        return 0;
 }
+
+
 /**
- * stripe_rename - 
  */
 int32_t
-stripe_rename (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *oldloc,
-	       loc_t *newloc)
+stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *xdata)
 {
-	stripe_private_t *priv = this->private;
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
+        int32_t           callcnt = 0;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        xlator_list_t    *trav = NULL;
+        call_frame_t     *prev = NULL;
 
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (oldloc);
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
 
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning ENOTCONN");
-		STACK_UNWIND (frame, -1, EIO, NULL);
-		return 0;
-	}
+        prev  = cookie;
+        priv  = this->private;
+        local = frame->local;
 
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	local->inode = oldloc->inode;
-	loc_copy (&local->loc, oldloc);
-	loc_copy (&local->loc2, newloc);
+	LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_ret = -1;
+                        local->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->op_ret == -1) {
+                        local->call_count = priv->child_count;
+                        while (trav) {
+                                STACK_WIND (frame,
+                                            stripe_mknod_ifreg_fail_unlink_cbk,
+                                            trav->xlator,
+                                            trav->xlator->fops->unlink,
+                                            &local->loc, 0, NULL);
+                                trav = trav->next;
+                        }
+                        return 0;
+                }
+
+                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
+                                     local->inode, &local->stbuf,
+                                     &local->preparent, &local->postparent, NULL);
+        }
+out:
+        return 0;
+}
 
-	local->call_count = priv->child_count;
-  
-	frame->local = local;
+int32_t
+stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
+{
+        int32_t           callcnt = 0;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        call_frame_t     *prev = NULL;
+        xlator_list_t    *trav = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        priv  = this->private;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                        local->op_errno = op_errno;
+                }
+                if (op_ret >= 0) {
+                        local->op_ret = op_ret;
+
+                        /* Can be used as a mechanism to understand if mknod
+                           was successful in at least one place */
+                        if (uuid_is_null (local->ia_gfid))
+                                uuid_copy (local->ia_gfid, buf->ia_gfid);
+
+			if (stripe_ctx_handle(this, prev, local, xdata))
+				gf_log(this->name, GF_LOG_ERROR,
+					"Error getting fctx info from dict");
+
+                        local->stbuf_blocks += buf->ia_blocks;
+                        local->preparent_blocks  += preparent->ia_blocks;
+                        local->postparent_blocks += postparent->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
+                        if (local->stbuf_size < buf->ia_size)
+                                local->stbuf_size = buf->ia_size;
+                        if (local->preparent_size < preparent->ia_size)
+                                local->preparent_size = preparent->ia_size;
+                        if (local->postparent_size < postparent->ia_size)
+                                local->postparent_size = postparent->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                if ((local->op_ret == -1) && !uuid_is_null (local->ia_gfid)) {
+                        /* ia_gfid set means, at least on one node 'mknod'
+                           is successful */
+                        local->call_count = priv->child_count;
+                        trav = this->children;
+                        while (trav) {
+                                STACK_WIND (frame,
+                                            stripe_mknod_ifreg_fail_unlink_cbk,
+                                            trav->xlator,
+                                            trav->xlator->fops->unlink,
+                                            &local->loc, 0, NULL);
+                                trav = trav->next;
+                        }
+                        return 0;
+                }
+
+
+                if (local->op_ret != -1) {
+                        local->preparent.ia_blocks  = local->preparent_blocks;
+                        local->preparent.ia_size    = local->preparent_size;
+                        local->postparent.ia_blocks = local->postparent_blocks;
+                        local->postparent.ia_size   = local->postparent_size;
+                        local->stbuf.ia_size        = local->stbuf_size;
+                        local->stbuf.ia_blocks      = local->stbuf_blocks;
+                        inode_ctx_put (local->inode, this,
+                                       (uint64_t)(long) local->fctx);
+
+                }
+                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
+                                     local->inode, &local->stbuf,
+                                     &local->preparent, &local->postparent, NULL);
+        }
+out:
+        return 0;
+}
 
-	STACK_WIND (frame,
-		    stripe_first_rename_cbk,
-		    trav->xlator,
-		    trav->xlator->fops->rename,
-		    oldloc, newloc);
 
-	return 0;
+int32_t
+stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        call_frame_t     *prev = NULL;
+        xlator_list_t    *trav = NULL;
+        int               i    = 1;
+        dict_t           *dict           = NULL;
+        int               ret            = 0;
+        int               need_unref     = 0;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        priv  = this->private;
+        local = frame->local;
+        trav = this->children;
+
+        local->call_count--;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
+                        prev->this->name, strerror (op_errno));
+                local->failed = 1;
+                local->op_errno = op_errno;
+                goto out;
+        }
+
+        local->op_ret = op_ret;
+
+        local->stbuf      = *buf;
+        local->preparent  = *preparent;
+        local->postparent = *postparent;
+
+        if (uuid_is_null (local->ia_gfid))
+                uuid_copy (local->ia_gfid, buf->ia_gfid);
+        local->preparent.ia_blocks  = local->preparent_blocks;
+        local->preparent.ia_size    = local->preparent_size;
+        local->postparent.ia_blocks = local->postparent_blocks;
+        local->postparent.ia_size   = local->postparent_size;
+        local->stbuf.ia_size        = local->stbuf_size;
+        local->stbuf.ia_blocks      = local->stbuf_blocks;
+
+        trav = trav->next;
+        while (trav) {
+                if (priv->xattr_supported) {
+                        dict = dict_new ();
+                        if (!dict) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to allocate dict %s", local->loc.path);
+                        }
+                        need_unref = 1;
+
+                        dict_copy (local->xattr, dict);
+
+                        ret = stripe_xattr_request_build (this, dict,
+                                                          local->stripe_size,
+                                                          priv->child_count, i,
+							  priv->coalesce);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to build xattr request");
+
+                } else {
+                        dict = local->xattr;
+                }
+
+                STACK_WIND (frame, stripe_mknod_ifreg_cbk,
+                            trav->xlator, trav->xlator->fops->mknod,
+                            &local->loc, local->mode, local->rdev, 0, dict);
+                trav = trav->next;
+                i++;
+
+                if (dict && need_unref)
+                        dict_unref (dict);
+        }
+
+        return 0;
+
+out:
+
+       STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL, NULL);
+       return 0;
 }
 
 
-/**
- * stripe_access - 
- */
 int32_t
-stripe_access (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       int32_t mask)
+stripe_single_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
+        STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
 
-	STACK_WIND (frame,
-		    stripe_common_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->access,
-		    loc, mask);
 
-	return 0;
+int
+stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        stripe_private_t *priv           = NULL;
+        stripe_local_t   *local          = NULL;
+        int32_t           op_errno       = EINVAL;
+        int32_t           i              = 0;
+        dict_t           *dict           = NULL;
+        int               ret            = 0;
+        int               need_unref     = 0;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        if (S_ISREG(mode)) {
+                /* NOTE: on older kernels (older than 2.6.9),
+                   creat() fops is sent as mknod() + open(). Hence handling
+                   S_IFREG files is necessary */
+                if (priv->nodes_down) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Some node down, returning EIO");
+                        op_errno = EIO;
+                        goto err;
+                }
+
+                /* Initialization */
+                local = mem_get0 (this->local_pool);
+                if (!local) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                local->op_ret = -1;
+                local->op_errno = ENOTCONN;
+                local->stripe_size = stripe_get_matching_bs (loc->path, priv);
+                frame->local = local;
+                local->inode = inode_ref (loc->inode);
+                loc_copy (&local->loc, loc);
+                local->xattr = dict_copy_with_ref (xdata, NULL);
+                local->mode = mode;
+                local->umask = umask;
+                local->rdev = rdev;
+
+                /* Everytime in stripe lookup, all child nodes should
+                   be looked up */
+                local->call_count = priv->child_count;
+
+                if (priv->xattr_supported) {
+                        dict = dict_new ();
+                        if (!dict) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to allocate dict %s", loc->path);
+                        }
+                        need_unref = 1;
+
+                        dict_copy (xdata, dict);
+
+                        ret = stripe_xattr_request_build (this, dict,
+                                                          local->stripe_size,
+                                                          priv->child_count,
+                                                          i, priv->coalesce);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to build xattr request");
+                } else {
+                        dict = xdata;
+                }
+
+                STACK_WIND (frame, stripe_mknod_first_ifreg_cbk,
+                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
+                            loc, mode, rdev, umask, dict);
+
+                        if (dict && need_unref)
+                                dict_unref (dict);
+                return 0;
+        }
+
+        STACK_WIND (frame, stripe_single_mknod_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                    loc, mode, rdev, umask, xdata);
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 
-/**
- * stripe_readlink_cbk - 
- */
-int32_t 
-stripe_readlink_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     const char *path)
+int32_t
+stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, path);
-
-	return 0;
+        int32_t         callcnt = 0;
+        stripe_local_t  *local   = NULL;
+        call_frame_t    *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+
+                if (op_ret >= 0) {
+                        local->op_ret = 0;
+
+                        local->stbuf_blocks      += buf->ia_blocks;
+                        local->preparent_blocks  += preparent->ia_blocks;
+                        local->postparent_blocks += postparent->ia_blocks;
+
+                        if (local->stbuf_size < buf->ia_size)
+                                local->stbuf_size = buf->ia_size;
+                        if (local->preparent_size < preparent->ia_size)
+                                local->preparent_size = preparent->ia_size;
+                        if (local->postparent_size < postparent->ia_size)
+                                local->postparent_size = postparent->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed != -1) {
+                        local->preparent.ia_blocks  = local->preparent_blocks;
+                        local->preparent.ia_size    = local->preparent_size;
+                        local->postparent.ia_blocks = local->postparent_blocks;
+                        local->postparent.ia_size   = local->postparent_size;
+                        local->stbuf.ia_size        = local->stbuf_size;
+                        local->stbuf.ia_blocks      = local->stbuf_blocks;
+                }
+                STRIPE_STACK_UNWIND (mkdir, frame, local->op_ret,
+                                     local->op_errno, local->inode,
+                                     &local->stbuf, &local->preparent,
+                                     &local->postparent, NULL);
+        }
+out:
+        return 0;
 }
 
 
-/**
- * stripe_readlink - 
- */
 int32_t
-stripe_readlink (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 size_t size)
+stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-	stripe_private_t *priv = this->private;
+        stripe_local_t  *local   = NULL;
+        call_frame_t    *prev = NULL;
+        xlator_list_t        *trav = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+        trav = this->children;
+
+        local->call_count--; /* first child is successful */
+        trav = trav->next;   /* skip first child */
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
+                        prev->this->name, strerror (op_errno));
+                local->op_errno = op_errno;
+                goto out;
+        }
+
+        local->op_ret = 0;
+
+        local->inode      = inode_ref (inode);
+        local->stbuf      = *buf;
+        local->postparent = *postparent;
+        local->preparent  = *preparent;
+
+        local->stbuf_blocks      += buf->ia_blocks;
+        local->preparent_blocks  += preparent->ia_blocks;
+        local->postparent_blocks += postparent->ia_blocks;
+
+        local->stbuf_size = buf->ia_size;
+        local->preparent_size = preparent->ia_size;
+        local->postparent_size = postparent->ia_size;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_mkdir_cbk, trav->xlator,
+                            trav->xlator->fops->mkdir, &local->loc, local->mode,
+                            local->umask, local->xdata);
+                trav = trav->next;
+        }
+        return 0;
+out:
+        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL);
+
+        return 0;
 
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning ENOTCONN");
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
-	}
+}
 
-	STACK_WIND (frame,
-		    stripe_readlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readlink,
-		    loc, size);
 
-	return 0;
+int
+stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              mode_t umask, dict_t *xdata)
+{
+        stripe_private_t *priv = NULL;
+        stripe_local_t   *local = NULL;
+        xlator_list_t    *trav = NULL;
+        int32_t           op_errno = 1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        local->call_count = priv->child_count;
+        if (xdata)
+                local->xdata = dict_ref (xdata);
+        local->mode  = mode;
+        local->umask = umask;
+        loc_copy (&local->loc, loc);
+        frame->local = local;
+
+        /* Everytime in stripe lookup, all child nodes should be looked up */
+        STACK_WIND (frame, stripe_first_mkdir_cbk, trav->xlator,
+                    trav->xlator->fops->mkdir, loc, mode, umask, xdata);
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 
-/**
- * stripe_unlink - 
- */
 int32_t
-stripe_unlink (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc)
+stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-	int send_fop_to_all = 0;
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
+        int32_t         callcnt = 0;
+        stripe_local_t  *local   = NULL;
+        call_frame_t    *prev = NULL;
+	stripe_fd_ctx_t *fctx = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+
+                if (op_ret >= 0) {
+                        local->op_ret = 0;
+
+			if (IA_ISREG(inode->ia_type)) {
+				inode_ctx_get(inode, this, (uint64_t *) &fctx);
+				if (!fctx) {
+					gf_log(this->name, GF_LOG_ERROR,
+						"failed to get stripe context");
+					op_ret = -1;
+					op_errno = EINVAL;
+				}
+			}
 
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
+                        if (FIRST_CHILD(this) == prev->this) {
+                                local->inode      = inode_ref (inode);
+                                local->stbuf      = *buf;
+                                local->postparent = *postparent;
+                                local->preparent  = *preparent;
+                        }
+                        local->stbuf_blocks      += buf->ia_blocks;
+                        local->preparent_blocks  += preparent->ia_blocks;
+                        local->postparent_blocks += postparent->ia_blocks;
+
+			correct_file_size(buf, fctx, prev);
+
+                        if (local->stbuf_size < buf->ia_size)
+                                local->stbuf_size = buf->ia_size;
+                        if (local->preparent_size < preparent->ia_size)
+                                local->preparent_size = preparent->ia_size;
+                        if (local->postparent_size < postparent->ia_size)
+                                local->postparent_size = postparent->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                if (local->op_ret != -1) {
+                        local->preparent.ia_blocks  = local->preparent_blocks;
+                        local->preparent.ia_size    = local->preparent_size;
+                        local->postparent.ia_blocks = local->postparent_blocks;
+                        local->postparent.ia_size   = local->postparent_size;
+                        local->stbuf.ia_size        = local->stbuf_size;
+                        local->stbuf.ia_blocks      = local->stbuf_blocks;
+                }
+                STRIPE_STACK_UNWIND (link, frame, local->op_ret,
+                                     local->op_errno, local->inode,
+                                     &local->stbuf, &local->preparent,
+                                     &local->postparent, NULL);
+        }
+out:
+        return 0;
+}
 
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning EIO");
-		STACK_UNWIND (frame, -1, EIO);
-		return 0;
-	}
- 
-	if (S_ISDIR (loc->inode->st_mode) || S_ISREG (loc->inode->st_mode))
-		send_fop_to_all = 1;
-
-	if (!send_fop_to_all) {
-		STACK_WIND (frame,
-			    stripe_common_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->unlink,
-			    loc);
-	} else {
-		/* Initialization */
-		local = CALLOC (1, sizeof (stripe_local_t));
-		ERR_ABORT (local);
-		local->op_ret = -1;
-		frame->local = local;
-		local->call_count = priv->child_count;
-    
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_stack_unwind_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->unlink,
-				    loc);
-			trav = trav->next;
-		}
-	}
+int32_t
+stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        xlator_list_t    *trav = NULL;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        int32_t           op_errno = 1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (oldloc, err);
+        VALIDATE_OR_GOTO (oldloc->path, err);
+        VALIDATE_OR_GOTO (oldloc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* If any one node is down, don't allow link operation */
+        if (priv->nodes_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+        /* Everytime in stripe lookup, all child
+           nodes should be looked up */
+        while (trav) {
+                STACK_WIND (frame, stripe_link_cbk,
+                            trav->xlator, trav->xlator->fops->link,
+                            oldloc, newloc, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        return 0;
+}
 
-	return 0;
+int32_t
+stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret,
+                               int32_t op_errno, struct iatt *preparent,
+                               struct iatt *postparent, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+
+        if (!this || !frame || !frame->local) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                STRIPE_STACK_UNWIND (create, frame, local->op_ret, local->op_errno,
+                                     local->fd, local->inode, &local->stbuf,
+                                     &local->preparent, &local->postparent, NULL);
+        }
+out:
+        return 0;
 }
 
 
-int32_t 
-stripe_first_rmdir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno)
+int32_t
+stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd,
+                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	xlator_list_t *trav = this->children;
-	stripe_local_t *local = frame->local;
+        int32_t           callcnt = 0;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        call_frame_t     *prev = NULL;
+        xlator_list_t    *trav = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        priv  = this->private;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->failed = 1;
+                        local->op_errno = op_errno;
+                }
+
+                if (op_ret >= 0) {
+			if (IA_ISREG(buf->ia_type)) {
+				if (stripe_ctx_handle(this, prev, local, xdata))
+					gf_log(this->name, GF_LOG_ERROR,
+						"Error getting fctx info from "
+						"dict");
+			}
 
-	if (op_ret == -1) 
-	{
-		STACK_UNWIND (frame, op_ret, op_errno);
-		return 0;
-	}
+                        local->op_ret = op_ret;
+
+                        local->stbuf_blocks += buf->ia_blocks;
+                        local->preparent_blocks  += preparent->ia_blocks;
+                        local->postparent_blocks += postparent->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
+                        if (local->stbuf_size < buf->ia_size)
+                                local->stbuf_size = buf->ia_size;
+                        if (local->preparent_size < preparent->ia_size)
+                                local->preparent_size = preparent->ia_size;
+                        if (local->postparent_size < postparent->ia_size)
+                                local->postparent_size = postparent->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                if (local->op_ret == -1) {
+                        local->call_count = priv->child_count;
+                        trav = this->children;
+                        while (trav) {
+                                STACK_WIND (frame,
+                                            stripe_create_fail_unlink_cbk,
+                                            trav->xlator,
+                                            trav->xlator->fops->unlink,
+                                            &local->loc, 0, NULL);
+                                trav = trav->next;
+                        }
+
+                        return 0;
+                }
+
+                if (local->op_ret >= 0) {
+                        local->preparent.ia_blocks  = local->preparent_blocks;
+                        local->preparent.ia_size    = local->preparent_size;
+                        local->postparent.ia_blocks = local->postparent_blocks;
+                        local->postparent.ia_size   = local->postparent_size;
+                        local->stbuf.ia_size        = local->stbuf_size;
+                        local->stbuf.ia_blocks      = local->stbuf_blocks;
+
+			stripe_copy_xl_array(local->fctx->xl_array,
+				             priv->xl_array,
+					     local->fctx->stripe_count);
+			inode_ctx_put(local->inode, this,
+					(uint64_t) local->fctx);
+                }
+
+                /* Create itself has failed.. so return
+                   without setxattring */
+                STRIPE_STACK_UNWIND (create, frame, local->op_ret,
+                                     local->op_errno, local->fd,
+                                     local->inode, &local->stbuf,
+                                     &local->preparent, &local->postparent, NULL);
+        }
+
+out:
+        return 0;
+}
 
-	local->call_count--; /* First child successful */
-	trav = trav->next; /* Skip first child */
 
-	while (trav) {
-		STACK_WIND (frame,
-			    stripe_stack_unwind_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->rmdir,
-			    &local->loc);
-		trav = trav->next;
-	}
 
-	return 0;
+int32_t
+stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd,
+                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        call_frame_t     *prev = NULL;
+        xlator_list_t    *trav = NULL;
+        int               i    = 1;
+        dict_t           *dict = NULL;
+        loc_t            *loc  = NULL;
+        int32_t           need_unref = 0;
+        int32_t           ret  = -1;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        priv  = this->private;
+        local = frame->local;
+        trav = this->children;
+        loc = &local->loc;
+
+        --local->call_count;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
+                        prev->this->name, strerror (op_errno));
+                 local->failed = 1;
+                 local->op_errno = op_errno;
+        }
+
+        local->op_ret = 0;
+        /* Get the mapping in inode private */
+        /* Get the stat buf right */
+        local->stbuf      = *buf;
+        local->preparent  = *preparent;
+        local->postparent = *postparent;
+
+        local->stbuf_blocks += buf->ia_blocks;
+        local->preparent_blocks  += preparent->ia_blocks;
+        local->postparent_blocks += postparent->ia_blocks;
+
+        if (local->stbuf_size < buf->ia_size)
+              local->stbuf_size = buf->ia_size;
+        if (local->preparent_size < preparent->ia_size)
+              local->preparent_size = preparent->ia_size;
+        if (local->postparent_size < postparent->ia_size)
+              local->postparent_size = postparent->ia_size;
+
+        if (local->failed)
+                local->op_ret = -1;
+
+        if (local->op_ret == -1) {
+                local->call_count = 1;
+                STACK_WIND (frame, stripe_create_fail_unlink_cbk,
+                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
+                            &local->loc, 0, NULL);
+                return 0;
+        }
+
+        if (local->op_ret >= 0) {
+                local->preparent.ia_blocks  = local->preparent_blocks;
+                local->preparent.ia_size    = local->preparent_size;
+                local->postparent.ia_blocks = local->postparent_blocks;
+                local->postparent.ia_size   = local->postparent_size;
+                local->stbuf.ia_size        = local->stbuf_size;
+                local->stbuf.ia_blocks      = local->stbuf_blocks;
+        }
+
+        /* Send a setxattr request to nodes where the
+           files are created */
+        trav = trav->next;
+        while (trav) {
+                if (priv->xattr_supported) {
+                        dict = dict_new ();
+                        if (!dict) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to allocate dict %s", loc->path);
+                        }
+                        need_unref = 1;
+
+                        dict_copy (local->xattr, dict);
+
+                        ret = stripe_xattr_request_build (this, dict,
+                                                          local->stripe_size,
+                                                          priv->child_count,
+                                                          i, priv->coalesce);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to build xattr request");
+                } else {
+                        dict = local->xattr;
+                }
+
+                STACK_WIND (frame, stripe_create_cbk, trav->xlator,
+                            trav->xlator->fops->create, &local->loc,
+                            local->flags, local->mode, local->umask, local->fd,
+                            dict);
+                trav = trav->next;
+                if (need_unref && dict)
+                        dict_unref (dict);
+                i++;
+        }
+
+out:
+        return 0;
 }
 
+
+
 /**
- * stripe_rmdir - 
+ * stripe_create - If a block-size is specified for the 'name', create the
+ *    file in all the child nodes. If not, create it in only first child.
+ *
+ * @name- complete path of the file to be created.
  */
 int32_t
-stripe_rmdir (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
+stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-	stripe_private_t *priv = this->private;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning EIO");
-		STACK_UNWIND (frame, -1, EIO);
-		return 0;
-	}
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-	local->inode = loc->inode;
-	loc_copy (&local->loc, loc);
-	local->call_count = priv->child_count;
-  
-	STACK_WIND (frame,
-		    stripe_first_rmdir_cbk,
-		    trav->xlator,
-		    trav->xlator->fops->rmdir,
-		    loc);
+        stripe_private_t *priv = NULL;
+        stripe_local_t   *local = NULL;
+        int32_t           op_errno = EINVAL;
+        int               ret            = 0;
+        int               need_unref     = 0;
+        int               i              = 0;
+        dict_t           *dict           = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+
+        /* files created in O_APPEND mode does not allow lseek() on fd */
+        flags &= ~O_APPEND;
+
+        if (priv->first_child_down || priv->nodes_down) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "First node down, returning EIO");
+                op_errno = EIO;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
+        frame->local = local;
+        local->inode = inode_ref (loc->inode);
+        loc_copy (&local->loc, loc);
+        local->fd = fd_ref (fd);
+        local->flags = flags;
+        local->mode = mode;
+        local->umask = umask;
+        if (xdata)
+                local->xattr = dict_ref (xdata);
+
+        local->call_count = priv->child_count;
+        /* Send a setxattr request to nodes where the
+           files are created */
+
+        if (priv->xattr_supported) {
+                dict = dict_new ();
+                if (!dict) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to allocate dict %s", loc->path);
+                }
+                need_unref = 1;
+
+                dict_copy (xdata, dict);
+
+                ret = stripe_xattr_request_build (this, dict,
+                                                  local->stripe_size,
+                                                  priv->child_count,
+                                                  i, priv->coalesce);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to build xattr request");
+        } else {
+                        dict = xdata;
+        }
+
+
+        STACK_WIND (frame, stripe_first_create_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->create, loc, flags, mode,
+                    umask, fd, dict);
+
+        if (need_unref && dict)
+                dict_unref (dict);
+
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL, xdata);
+        return 0;
+}
 
-	return 0;
+int32_t
+stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                        local->op_errno = op_errno;
+                }
+
+                if (op_ret >= 0)
+                        local->op_ret = op_ret;
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                STRIPE_STACK_UNWIND (open, frame, local->op_ret,
+                                     local->op_errno, local->fd, xdata);
+        }
+out:
+        return 0;
 }
 
 
 /**
- * stripe_setxattr - 
+ * stripe_open -
  */
 int32_t
-stripe_setxattr (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 dict_t *dict,
-		 int32_t flags)
+stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             int32_t flags, fd_t *fd, dict_t *xdata)
 {
-	stripe_private_t *priv = this->private;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        xlator_list_t    *trav = NULL;
+        int32_t           op_errno = 1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        /* files opened in O_APPEND mode does not allow lseek() on fd */
+        flags &= ~O_APPEND;
+
+        local->fd = fd_ref (fd);
+        frame->local = local;
+        loc_copy (&local->loc, loc);
+
+        /* Striped files */
+        local->flags = flags;
+        local->call_count = priv->child_count;
+        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
+
+        while (trav) {
+                STACK_WIND (frame, stripe_open_cbk, trav->xlator,
+                            trav->xlator->fops->open,
+                            &local->loc, local->flags, local->fd,
+                            xdata);
+                trav = trav->next;
+        }
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
 
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
 
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning ENOTCONN");
-		STACK_UNWIND (frame, -1, ENOTCONN);
-		return 0;
-	}
+int32_t
+stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_ret = -1;
+                        local->op_errno = op_errno;
+                }
+
+                if (op_ret >= 0)
+                        local->op_ret = op_ret;
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                STRIPE_STACK_UNWIND (opendir, frame, local->op_ret,
+                                     local->op_errno, local->fd, NULL);
+        }
+out:
+        return 0;
+}
 
-	STACK_WIND (frame,
-		    stripe_common_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setxattr,
-		    loc, dict, flags);
 
-	return 0;
+int32_t
+stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+        xlator_list_t    *trav = NULL;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        int32_t           op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->call_count = priv->child_count;
+        local->fd = fd_ref (fd);
+
+        while (trav) {
+                STACK_WIND (frame, stripe_opendir_cbk, trav->xlator,
+                            trav->xlator->fops->opendir, loc, fd, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
+        return 0;
 }
 
+int32_t
+stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+                if (op_ret >= 0) {
+                        if (FIRST_CHILD(this) == prev->this) {
+                                /* First successful call, copy the *lock */
+                                local->op_ret = op_ret;
+                                local->lock = *lock;
+                        }
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+                STRIPE_STACK_UNWIND (lk, frame, local->op_ret,
+                                     local->op_errno, &local->lock, NULL);
+        }
+out:
+        return 0;
+}
 
-int32_t 
-stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame,
-				    void *cookie,
-				    xlator_t *this,
-				    int32_t op_ret,
-				    int32_t op_errno)
+int32_t
+stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+           struct gf_flock *lock, dict_t *xdata)
 {
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
+        stripe_local_t   *local = NULL;
+        xlator_list_t    *trav = NULL;
+        stripe_private_t *priv = NULL;
+        int32_t           op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        trav = this->children;
+        priv = this->private;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_lk_cbk, trav->xlator,
+                            trav->xlator->fops->lk, fd, cmd, lock, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
 
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
 
-	if (!callcnt) {
-		loc_wipe (&local->loc);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local->inode, &local->stbuf);
-	}
+int32_t
+stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local   = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+                if (op_ret >= 0)
+                        local->op_ret = op_ret;
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                STRIPE_STACK_UNWIND (flush, frame, local->op_ret,
+                                     local->op_errno, NULL);
+        }
+out:
+        return 0;
+}
 
-	return 0;
+int32_t
+stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        xlator_list_t    *trav = NULL;
+        int32_t           op_errno = 1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_flush_cbk,  trav->xlator,
+                            trav->xlator->fops->flush, fd, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
+        return 0;
 }
 
 
-/**
- */
+
 int32_t
-stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno)
+stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
+        int32_t         callcnt = 0;
+        stripe_local_t *local   = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+                if (op_ret >= 0) {
+                        local->op_ret = op_ret;
+                        if (FIRST_CHILD(this) == prev->this) {
+                                local->pre_buf  = *prebuf;
+                                local->post_buf = *postbuf;
+                        }
+                        local->prebuf_blocks  += prebuf->ia_blocks;
+                        local->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, local->fctx, prev);
+			correct_file_size(postbuf, local->fctx, prev);
+
+                        if (local->prebuf_size < prebuf->ia_size)
+                                local->prebuf_size = prebuf->ia_size;
+
+                        if (local->postbuf_size < postbuf->ia_size)
+                                local->postbuf_size = postbuf->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                if (local->op_ret != -1) {
+                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
+                        local->pre_buf.ia_size    = local->prebuf_size;
+                        local->post_buf.ia_blocks = local->postbuf_blocks;
+                        local->post_buf.ia_size   = local->postbuf_size;
+                }
+
+                STRIPE_STACK_UNWIND (fsync, frame, local->op_ret,
+                                     local->op_errno, &local->pre_buf,
+                                     &local->post_buf, NULL);
+        }
+out:
+        return 0;
+}
 
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_ret = -1;
-			local->op_errno = op_errno;
-		}
+int32_t
+stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		op_errno = EINVAL;
+		goto err;
 	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		if (local->op_ret == -1) {
-			local->call_count = priv->child_count;
-			while (trav) {
-				STACK_WIND (frame,
-					    stripe_mknod_ifreg_fail_unlink_cbk,
-					    trav->xlator,
-					    trav->xlator->fops->unlink,
-					    &local->loc);
-				trav = trav->next;
-			}
-			return 0;
-		}
+	local->fctx = fctx;
+
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_fsync_cbk, trav->xlator,
+                            trav->xlator->fops->fsync, fd, flags, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
-		loc_wipe (&local->loc);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local->inode, &local->stbuf);
-	}
-	return 0;
+int32_t
+stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+
+                if (op_ret == 0) {
+                        local->op_ret = 0;
+
+                        if (FIRST_CHILD(this) == prev->this)
+                                local->stbuf = *buf;
+
+                        local->stbuf_blocks += buf->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
+                        if (local->stbuf_size < buf->ia_size)
+                                local->stbuf_size = buf->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                if (local->op_ret != -1) {
+                        local->stbuf.ia_size   = local->stbuf_size;
+                        local->stbuf.ia_blocks = local->stbuf_blocks;
+                }
+
+                STRIPE_STACK_UNWIND (fstat, frame, local->op_ret,
+                                     local->op_errno, &local->stbuf, NULL);
+        }
+
+out:
+        return 0;
 }
 
-/**
- */
 int32_t
-stripe_mknod_ifreg_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			inode_t *inode,
-			struct stat *buf)
+stripe_fstat (call_frame_t *frame,
+              xlator_t *this,
+              fd_t *fd, dict_t *xdata)
 {
-	int ret = 0;
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
-	stripe_private_t *priv = this->private;
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+	if (IA_ISREG(fd->inode->ia_type)) {
+		inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
 
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->failed = 1;
-			local->op_errno = op_errno;
-		}
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			/* Get the mapping in inode private */
-			/* Get the stat buf right */
-			if (local->stbuf.st_blksize == 0) {
-				local->stbuf = *buf;
-				/* Because st_blocks gets added again */
-				local->stbuf.st_blocks = 0;
-			}
+        while (trav) {
+                STACK_WIND (frame, stripe_fstat_cbk, trav->xlator,
+                            trav->xlator->fops->fstat, fd, NULL);
+                trav = trav->next;
+        }
 
-			/* Always, pass the inode number of first child 
-			   to the above layer */
-			if (FIRST_CHILD(this) == 
-			    ((call_frame_t *)cookie)->this)
-				local->stbuf.st_ino = buf->st_ino;
-      
-			local->stbuf.st_blocks += buf->st_blocks;
-			if (local->stbuf.st_size < buf->st_size)
-				local->stbuf.st_size = buf->st_size;
-			if (local->stbuf.st_blksize != buf->st_blksize) {
-				/* TODO: add to blocks in terms of
-				   original block size */
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		if (local->failed) 
-			local->op_ret = -1;
-
-		if ((local->op_ret != -1) && priv->xattr_supported) {
-			/* Send a setxattr request to nodes where the
-			   files are created */
-			int32_t index = 0;
-			char size_key[256] = {0,};
-			char index_key[256] = {0,};
-			char count_key[256] = {0,};
-			xlator_list_t *trav = this->children;
-			dict_t *dict = NULL;
-
-			sprintf (size_key, 
-				 "trusted.%s.stripe-size", this->name);
-			sprintf (count_key, 
-				 "trusted.%s.stripe-count", this->name);
-			sprintf (index_key, 
-				 "trusted.%s.stripe-index", this->name);
-
-			local->call_count = priv->child_count;
-
-			while (trav) {
-				dict = get_new_dict ();
-				dict_ref (dict);
-				/* TODO: check return value */
-				ret = dict_set_int64 (dict, size_key, 
-						local->stripe_size);
-				ret = dict_set_int32 (dict, count_key, 
-						local->call_count);
-				ret = dict_set_int32 (dict, index_key, index);
-
-				STACK_WIND (frame,
-					    stripe_mknod_ifreg_setxattr_cbk,
-					    trav->xlator,
-					    trav->xlator->fops->setxattr,
-					    &local->loc, dict, 0);
-	
-				dict_unref (dict);
-				index++;
-				trav = trav->next;
-			}
-		} else {
-			/* Create itself has failed.. so return 
-			   without setxattring */
-			loc_wipe (&local->loc);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      local->inode, &local->stbuf);
-		}
-	}
-  
-	return 0;
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+        return 0;
 }
 
 
-/**
- * stripe_mknod - 
- */
 int32_t
-stripe_mknod (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      mode_t mode,
-	      dev_t rdev)
-{
-	stripe_private_t *priv = this->private;
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = NULL;	
-  
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning EIO");
-		STACK_UNWIND (frame, -1, EIO, NULL, NULL);
-		return 0;
+stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
+	int		  i, eof_idx;
+	off_t		  dest_offset, tmp_offset;
+        int32_t		  op_errno = 1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        priv = this->private;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
+		op_errno = EINVAL;
+		goto err;
 	}
+	if (!fctx->stripe_count) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe count");
+		op_errno = EINVAL;
+		goto err;
+	}
+
+	local->fctx = fctx;
+	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
 
-	if (S_ISREG(mode)) {
-		/* NOTE: on older kernels (older than 2.6.9), 
-		   creat() fops is sent as mknod() + open(). Hence handling 
-		   S_IFREG files is necessary */
-		if (priv->nodes_down) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Some node down, returning EIO");
-			STACK_UNWIND (frame, -1, EIO, loc->inode, NULL);
-			return 0;
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (!fctx->xl_array[i]) {
+			gf_log(this->name, GF_LOG_ERROR, "no xlator at index "
+				"%d", i);
+			op_errno = EINVAL;
+			goto err;
 		}
-		
-		/* Initialization */
-		local = CALLOC (1, sizeof (stripe_local_t));
-		ERR_ABORT (local);
-		local->op_ret = -1;
-		local->op_errno = ENOTCONN;
-		local->stripe_size = stripe_get_matching_bs (loc->path,
-							     priv->pattern,
-							     priv->block_size);
-		frame->local = local;
-		local->inode = loc->inode;
-		loc_copy (&local->loc, loc);
-
-		/* Everytime in stripe lookup, all child nodes should
-		   be looked up */
-		local->call_count = 
-			((stripe_private_t *)this->private)->child_count;
-		
-		trav = this->children;
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_mknod_ifreg_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->mknod,
-				    loc, mode, rdev);
-			trav = trav->next;
+
+		if (fctx->stripe_coalesce) {
+			if (i < eof_idx)
+				tmp_offset = roof(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else if (i > eof_idx)
+				tmp_offset = floor(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else
+				tmp_offset = offset;
+
+			dest_offset = coalesced_offset(tmp_offset,
+				fctx->stripe_size, fctx->stripe_count);
+		} else {
+			dest_offset = offset;
 		}
 
-		/* This case is handled, no need to continue further. */
-		return 0; 
+		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
+			fctx->xl_array[i]->fops->ftruncate, fd, dest_offset,
+			NULL);
 	}
 
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
-	STACK_WIND (frame,
-		    stripe_common_inode_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev);
 
-	return 0;
+int32_t
+stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local   = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s returned %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        if ((op_errno != ENOENT) ||
+                            (prev->this == FIRST_CHILD (this)))
+                                local->failed = 1;
+                }
+                if (op_ret >= 0)
+                        local->op_ret = op_ret;
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
+                STRIPE_STACK_UNWIND (fsyncdir, frame, local->op_ret,
+                                     local->op_errno, NULL);
+        }
+out:
+        return 0;
 }
 
-
-/**
- * stripe_mkdir - 
- */
 int32_t
-stripe_mkdir (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      mode_t mode)
-{
-	stripe_private_t *priv = this->private;
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = NULL;
-  
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning EIO");
-		STACK_UNWIND (frame, -1, EIO, NULL, NULL);
-		return 0;
-	}
+stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        xlator_list_t    *trav = NULL;
+        int32_t           op_errno = 1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        local->call_count = priv->child_count;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_fsyncdir_cbk, trav->xlator,
+                            trav->xlator->fops->fsyncdir, fd, flags, NULL);
+                trav = trav->next;
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+        return 0;
+}
 
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	local->call_count = priv->child_count;
-	frame->local = local;
-
-	/* Everytime in stripe lookup, all child nodes should be looked up */
-	trav = this->children;
-	while (trav) {
-		STACK_WIND (frame,
-			    stripe_stack_unwind_inode_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->mkdir,
-			    loc, mode);
-		trav = trav->next;
-	}
 
-	return 0;
+int32_t
+stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        int32_t         i = 0;
+        int32_t         callcnt = 0;
+        int32_t         count = 0;
+        stripe_local_t *local = NULL;
+        struct iovec   *vec = NULL;
+        struct iatt     tmp_stbuf = {0,};
+        struct iobref  *tmp_iobref = NULL;
+        struct iobuf   *iobuf = NULL;
+	call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+	prev = cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                if (op_ret != -1) {
+			correct_file_size(buf, local->fctx, prev);
+                        if (local->stbuf_size < buf->ia_size)
+                                local->stbuf_size = buf->ia_size;
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                op_ret = 0;
+
+                /* Keep extra space for filling in '\0's */
+                vec = GF_CALLOC ((local->count * 2), sizeof (struct iovec),
+                                 gf_stripe_mt_iovec);
+                if (!vec) {
+                        op_ret = -1;
+                        goto done;
+                }
+
+                for (i = 0; i < local->wind_count; i++) {
+                        if (local->replies[i].op_ret) {
+                                memcpy ((vec + count), local->replies[i].vector,
+                                        (local->replies[i].count * sizeof (struct iovec)));
+                                count +=  local->replies[i].count;
+                                op_ret += local->replies[i].op_ret;
+                        }
+                        if ((local->replies[i].op_ret <
+                             local->replies[i].requested_size) &&
+                            (local->stbuf_size > (local->offset + op_ret))) {
+                                /* Fill in 0s here */
+                                vec[count].iov_len  =
+                                        (local->replies[i].requested_size -
+                                         local->replies[i].op_ret);
+                                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
+                                                    vec[count].iov_len);
+                                if (!iobuf) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Out of memory.");
+                                        op_ret = -1;
+                                        op_errno = ENOMEM;
+                                        goto done;
+                                }
+                                memset (iobuf->ptr, 0, vec[count].iov_len);
+                                vec[count].iov_base = iobuf->ptr;
+
+                                iobref_add (local->iobref, iobuf);
+                                iobuf_unref(iobuf);
+
+                                op_ret += vec[count].iov_len;
+                                count++;
+                        }
+                        GF_FREE (local->replies[i].vector);
+                }
+
+                /* FIXME: notice that st_ino, and st_dev (gen) will be
+                 * different than what inode will have. Make sure this doesn't
+                 * cause any bugs at higher levels */
+                memcpy (&tmp_stbuf, &local->replies[0].stbuf,
+                        sizeof (struct iatt));
+                tmp_stbuf.ia_size = local->stbuf_size;
+
+        done:
+                GF_FREE (local->replies);
+                tmp_iobref = local->iobref;
+                STRIPE_STACK_UNWIND (readv, frame, op_ret, op_errno, vec,
+                                     count, &tmp_stbuf, tmp_iobref, NULL);
+
+                iobref_unref (tmp_iobref);
+                GF_FREE (vec);
+        }
+out:
+        return 0;
 }
 
-
 /**
- * stripe_symlink - 
+ * stripe_readv_cbk - get all the striped reads, and order it properly, send it
+ *        to above layer after putting it in a single vector.
  */
 int32_t
-stripe_symlink (call_frame_t *frame,
-		xlator_t *this,
-		const char *linkpath,
-		loc_t *loc)
-{
-	stripe_private_t *priv = this->private;
-  
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning EIO");
-		STACK_UNWIND (frame, -1, EIO, NULL, NULL);
-		return 0;
-	}
+stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+        int32_t         index = 0;
+        int32_t         callcnt = 0;
+        int32_t         final_count = 0;
+        int32_t         need_to_check_proper_size = 0;
+        call_frame_t   *mframe = NULL;
+        stripe_local_t *mlocal = NULL;
+        stripe_local_t *local = NULL;
+        struct iovec   *final_vec = NULL;
+        struct iatt     tmp_stbuf = {0,};
+        struct iatt    *tmp_stbuf_p = NULL; //need it for a warning
+        struct iobref  *tmp_iobref = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+	call_frame_t	*prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto end;
+        }
+
+        local  = frame->local;
+        index  = local->node_index;
+	prev = cookie;
+        mframe = local->orig_frame;
+        if (!mframe)
+                goto out;
+
+        mlocal = mframe->local;
+        if (!mlocal)
+                goto out;
+
+        fctx = mlocal->fctx;
+
+        LOCK (&mframe->lock);
+        {
+                mlocal->replies[index].op_ret = op_ret;
+                mlocal->replies[index].op_errno = op_errno;
+                mlocal->replies[index].requested_size = local->readv_size;
+                if (op_ret >= 0) {
+                        mlocal->replies[index].stbuf  = *stbuf;
+                        mlocal->replies[index].count  = count;
+                        mlocal->replies[index].vector = iov_dup (vector, count);
+
+			correct_file_size(stbuf, fctx, prev);
+
+                        if (local->stbuf_size < stbuf->ia_size)
+                                local->stbuf_size = stbuf->ia_size;
+                        local->stbuf_blocks += stbuf->ia_blocks;
+
+                        if (!mlocal->iobref)
+                                mlocal->iobref = iobref_new ();
+                        iobref_merge (mlocal->iobref, iobref);
+                }
+                callcnt = ++mlocal->call_count;
+        }
+        UNLOCK(&mframe->lock);
+
+        if (callcnt == mlocal->wind_count) {
+                op_ret = 0;
+
+                for (index=0; index < mlocal->wind_count; index++) {
+                        /* check whether each stripe returned
+                         * 'expected' number of bytes */
+                        if (mlocal->replies[index].op_ret == -1) {
+                                op_ret = -1;
+                                op_errno = mlocal->replies[index].op_errno;
+                                break;
+                        }
+                        /* TODO: handle the 'holes' within the read range
+                           properly */
+                        if (mlocal->replies[index].op_ret <
+                            mlocal->replies[index].requested_size) {
+                                need_to_check_proper_size = 1;
+                        }
+
+                        op_ret       += mlocal->replies[index].op_ret;
+                        mlocal->count += mlocal->replies[index].count;
+                }
+                if (op_ret == -1)
+                        goto done;
+                if (need_to_check_proper_size)
+                        goto check_size;
+
+                final_vec = GF_CALLOC (mlocal->count, sizeof (struct iovec),
+                                       gf_stripe_mt_iovec);
+
+                if (!final_vec) {
+                        op_ret = -1;
+                        goto done;
+                }
+
+                for (index = 0; index < mlocal->wind_count; index++) {
+                        memcpy ((final_vec + final_count),
+                                mlocal->replies[index].vector,
+                                (mlocal->replies[index].count *
+                                 sizeof (struct iovec)));
+                        final_count +=  mlocal->replies[index].count;
+                        GF_FREE (mlocal->replies[index].vector);
+                }
+
+                /* FIXME: notice that st_ino, and st_dev (gen) will be
+                 * different than what inode will have. Make sure this doesn't
+                 * cause any bugs at higher levels */
+                memcpy (&tmp_stbuf, &mlocal->replies[0].stbuf,
+                        sizeof (struct iatt));
+                tmp_stbuf.ia_size = local->stbuf_size;
+                tmp_stbuf.ia_blocks = local->stbuf_blocks;
+
+        done:
+                /* */
+                GF_FREE (mlocal->replies);
+                tmp_iobref = mlocal->iobref;
+                /* work around for nfs truncated read. Bug 3774 */
+                tmp_stbuf_p = &tmp_stbuf;
+                WIPE (tmp_stbuf_p);
+                STRIPE_STACK_UNWIND (readv, mframe, op_ret, op_errno, final_vec,
+                                     final_count, &tmp_stbuf, tmp_iobref, NULL);
+
+                iobref_unref (tmp_iobref);
+                GF_FREE (final_vec);
+        }
+
+        goto out;
+
+check_size:
+        mlocal->call_count = fctx->stripe_count;
+
+        for (index = 0; index < fctx->stripe_count; index++) {
+                STACK_WIND (mframe, stripe_readv_fstat_cbk,
+                            (fctx->xl_array[index]),
+                            (fctx->xl_array[index])->fops->fstat,
+                            mlocal->fd, NULL);
+        }
+
+out:
+        STRIPE_STACK_DESTROY (frame);
+end:
+        return 0;
+}
 
-	/* send symlink to only first node */
-	STACK_WIND (frame,
-		    stripe_common_inode_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc);
 
-	return 0;
+int32_t
+stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        int32_t           op_errno = EINVAL;
+        int32_t           idx = 0;
+        int32_t           index = 0;
+        int32_t           num_stripe = 0;
+        int32_t           off_index = 0;
+        size_t            frame_size = 0;
+        off_t             rounded_end = 0;
+        uint64_t          tmp_fctx = 0;
+        uint64_t          stripe_size = 0;
+        off_t             rounded_start = 0;
+        off_t             frame_offset = offset;
+	off_t		  dest_offset = 0;
+        stripe_local_t   *local = NULL;
+        call_frame_t     *rframe = NULL;
+        stripe_local_t   *rlocal = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EBADFD;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                goto err;
+        }
+        /* The file is stripe across the child nodes. Send the read request
+         * to the child nodes appropriately after checking which region of
+         * the file is in which child node. Always '0-<stripe_size>' part of
+         * the file resides in the first child.
+         */
+        rounded_start = floor (offset, stripe_size);
+        rounded_end = roof (offset+size, stripe_size);
+        num_stripe = (rounded_end- rounded_start)/stripe_size;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+
+        /* This is where all the vectors should be copied. */
+        local->replies = GF_CALLOC (num_stripe, sizeof (struct stripe_replies),
+                                    gf_stripe_mt_stripe_replies);
+        if (!local->replies) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        off_index = (offset / stripe_size) % fctx->stripe_count;
+        local->wind_count = num_stripe;
+        local->readv_size = size;
+        local->offset     = offset;
+        local->fd         = fd_ref (fd);
+        local->fctx       = fctx;
+
+        for (index = off_index; index < (num_stripe + off_index); index++) {
+                rframe = copy_frame (frame);
+                rlocal = mem_get0 (this->local_pool);
+                if (!rlocal) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+
+                frame_size = min (roof (frame_offset+1, stripe_size),
+                                  (offset + size)) - frame_offset;
+
+                rlocal->node_index = index - off_index;
+                rlocal->orig_frame = frame;
+                rlocal->readv_size = frame_size;
+                rframe->local = rlocal;
+                idx = (index % fctx->stripe_count);
+
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(frame_offset,
+				stripe_size, fctx->stripe_count);
+		else
+			dest_offset = frame_offset;
+
+                STACK_WIND (rframe, stripe_readv_cbk, fctx->xl_array[idx],
+                            fctx->xl_array[idx]->fops->readv,
+                            fd, frame_size, dest_offset, flags, xdata);
+
+                frame_offset += frame_size;
+        }
+
+        return 0;
+err:
+        if (rframe)
+                STRIPE_STACK_DESTROY (rframe);
+
+        STRIPE_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+        return 0;
 }
 
-/**
- * stripe_link -
- */
+
 int32_t
-stripe_link (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *oldloc,
-	     loc_t *newloc)
-{
-	int send_fop_to_all = 0;
-	stripe_private_t *priv = this->private;
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-  
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (oldloc);
-
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning EIO");
-		STACK_UNWIND (frame, -1, EIO, NULL, NULL);
-		return 0;
-	}
+stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+	struct stripe_replies *reply = NULL;
+	int32_t		i = 0;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+		mlocal->replies[local->node_index].op_ret = op_ret;
+		mlocal->replies[local->node_index].op_errno = op_errno;
+
+                if (op_ret >= 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+		/*
+		 * Only return the number of consecutively written bytes up until
+		 * the first error. Only return an error if it occurs first.
+		 *
+		 * When a short write occurs, the application should retry at the
+		 * appropriate offset, at which point we'll potentially pass back
+		 * the error.
+		 */
+		for (i = 0, reply = mlocal->replies; i < mlocal->wind_count;
+			i++, reply++) {
+			if (reply->op_ret == -1) {
+				gf_log(this->name, GF_LOG_DEBUG, "reply %d "
+					"returned error %s", i,
+					strerror(reply->op_errno));
+				if (!mlocal->op_ret) {
+					mlocal->op_ret = -1;
+					mlocal->op_errno = reply->op_errno;
+				}
+				break;
+			}
 
+			mlocal->op_ret += reply->op_ret;
 
-	if (S_ISREG (oldloc->inode->st_mode))
-		send_fop_to_all = 1;
-
-	if (!send_fop_to_all) {
-		STACK_WIND (frame,
-			    stripe_common_inode_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->link,
-			    oldloc, newloc);
-	} else {
-		/* Initialization */
-		local = CALLOC (1, sizeof (stripe_local_t));
-		ERR_ABORT (local);
-		local->op_ret = -1;
-		frame->local = local;
-		local->call_count = priv->child_count;
-
-		/* Everytime in stripe lookup, all child
-		   nodes should be looked up */
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_stack_unwind_inode_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->link,
-				    oldloc, newloc);
-			trav = trav->next;
+			if (reply->op_ret < reply->requested_size)
+				break;
 		}
-	}
 
-	return 0;
+		GF_FREE(mlocal->replies);
+
+                STRIPE_STACK_UNWIND (writev, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
 }
 
-int32_t 
-stripe_create_fail_unlink_cbk (call_frame_t *frame,
-			       void *cookie,
-			       xlator_t *this,
-			       int32_t op_ret,
-			       int32_t op_errno)
+int32_t
+stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t offset,
+               uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-	int32_t callcnt = 0;
-	fd_t *lfd = NULL;
-	stripe_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		lfd = local->fd;
-		loc_wipe (&local->loc);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local->fd, local->inode, &local->stbuf);
-		fd_unref (lfd);
+        struct iovec     *tmp_vec = NULL;
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           total_size = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        int32_t           tmp_count = count;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	off_t		  rounded_start = 0;
+	off_t		  rounded_end = 0;
+	int32_t		  total_chunks = 0;
+	call_frame_t	  *wframe = NULL;
+	stripe_local_t	  *wlocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        /* File has to be stripped across the child nodes */
+        for (idx = 0; idx< count; idx ++) {
+                total_size += vector[idx].iov_len;
+        }
+        remaining_size = total_size;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+	rounded_start = floor(offset, stripe_size);
+	rounded_end = roof(offset + total_size, stripe_size);
+	total_chunks = (rounded_end - rounded_start) / stripe_size;
+	local->replies = GF_CALLOC(total_chunks, sizeof(struct stripe_replies),
+				gf_stripe_mt_stripe_replies);
+	if (!local->replies) {
+		op_errno = ENOMEM;
+		goto err;
 	}
-	return 0;
+
+	total_chunks = 0;
+        while (1) {
+		wframe = copy_frame(frame);
+		wlocal = mem_get0(this->local_pool);
+		if (!wlocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		wlocal->orig_frame = frame;
+		wframe->local = wlocal;
+
+                /* Send striped chunk of the vector to child
+                   nodes appropriately. */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                tmp_count = iov_subset (vector, count, offset_offset,
+                                        offset_offset + fill_size, NULL);
+                tmp_vec = GF_CALLOC (tmp_count, sizeof (struct iovec),
+                                     gf_stripe_mt_iovec);
+                if (!tmp_vec) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                tmp_count = iov_subset (vector, count, offset_offset,
+                                        offset_offset + fill_size, tmp_vec);
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		/*
+		 * Store off the request index (with respect to the chunk of the
+		 * initial offset) and the size of the request. This is required
+		 * in the callback to calculate an appropriate return value in
+		 * the event of a write failure in one or more requests.
+		 */
+		wlocal->node_index = total_chunks;
+		local->replies[total_chunks].requested_size = fill_size;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+                STACK_WIND (wframe, stripe_writev_cbk, fctx->xl_array[idx],
+                            fctx->xl_array[idx]->fops->writev, fd, tmp_vec,
+                            tmp_count, dest_offset, flags, iobref,
+                            xdata);
+
+                GF_FREE (tmp_vec);
+                offset_offset += fill_size;
+		total_chunks++;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (wframe)
+		STRIPE_STACK_DESTROY(wframe);
+
+        STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
 }
 
 
-/**
- * stripe_create_setxattr_cbk - 
- */
 int32_t
-stripe_create_setxattr_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno)
+stripe_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-	fd_t *lfd = NULL;
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_ret = -1;
-			local->op_errno = op_errno;
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
 		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		if (local->op_ret == -1) {
-			local->call_count = priv->child_count;
-			while (trav) {
-				STACK_WIND (frame,
-					    stripe_create_fail_unlink_cbk,
-					    trav->xlator,
-					    trav->xlator->fops->unlink,
-					    &local->loc);
-				trav = trav->next;
-			}
-	
-			return 0;
-		}
-
-		lfd = local->fd;
-		loc_wipe (&local->loc);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			      local->fd, local->inode, &local->stbuf);
-		fd_unref (lfd);
-	}
-
-	return 0;
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (fallocate, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
 }
 
-/**
- * stripe_create_cbk - 
- */
 int32_t
-stripe_create_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd,
-		   inode_t *inode,
-		   struct stat *buf)
+stripe_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
 {
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
-	stripe_private_t *priv = this->private;
-	fd_t *lfd = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->failed = 1;
-			local->op_errno = op_errno;
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
 		}
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			/* Get the mapping in inode private */
-			/* Get the stat buf right */
-			if (local->stbuf.st_blksize == 0) {
-				local->stbuf = *buf;
-				/* Because st_blocks gets added again */
-				local->stbuf.st_blocks = 0;
-			}
-      
-			/* Always, pass the inode number of first
-			   child to the above layer */
-			if (FIRST_CHILD(this) == 
-			    ((call_frame_t *)cookie)->this)
-				local->stbuf.st_ino = buf->st_ino;
-      
-			local->stbuf.st_blocks += buf->st_blocks;
-			if (local->stbuf.st_size < buf->st_size)
-				local->stbuf.st_size = buf->st_size;
-			if (local->stbuf.st_blksize != buf->st_blksize) {
-				/* TODO: add to blocks in terms of 
-				   original block size */
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send fallocate request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single fallocate per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_fallocate_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->fallocate, fd, mode,
+			   dest_offset, fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
-	if (!callcnt) {
-		if (local->failed)
-			local->op_ret = -1;
 
-		if (local->op_ret >= 0) {
-			fd_ctx_set (local->fd, this, local->stripe_size);
+int32_t
+stripe_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
 		}
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (discard, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
 
-		if ((local->op_ret != -1) && 
-		    local->stripe_size && priv->xattr_supported) {
-			/* Send a setxattr request to nodes where
-			   the files are created */
-			int ret = 0;
-			int32_t index = 0;
-			char size_key[256] = {0,};
-			char index_key[256] = {0,};
-			char count_key[256] = {0,};
-			xlator_list_t *trav = this->children;
-			dict_t *dict = NULL;
-
-			sprintf (size_key, 
-				 "trusted.%s.stripe-size", this->name);
-			sprintf (count_key, 
-				 "trusted.%s.stripe-count", this->name);
-			sprintf (index_key,
-				 "trusted.%s.stripe-index", this->name);
-
-			local->call_count = priv->child_count;
-	
-			while (trav) {
-				dict = get_new_dict ();
-				dict_ref (dict);
-
-				/* TODO: check return values */
-				ret = dict_set_int64 (dict, size_key, 
-						      local->stripe_size);
-				ret = dict_set_int32 (dict, count_key, 
-						      local->call_count);
-				ret = dict_set_int32 (dict, index_key, index);
-	
-				STACK_WIND (frame,
-					    stripe_create_setxattr_cbk,
-					    trav->xlator,
-					    trav->xlator->fops->setxattr,
-					    &local->loc,
-					    dict,
-					    0);
-	
-				dict_unref (dict);
-				index++;
-				trav = trav->next;
-			}
-		} else {
-			/* Create itself has failed.. so return
-			   without setxattring */
-			lfd = local->fd;
-			loc_wipe (&local->loc);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      local->fd, local->inode, &local->stbuf);
-      
-			fd_unref (lfd);
+int32_t
+stripe_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
 		}
-	}
-  
-	return 0;
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send discard request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single discard per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_discard_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->discard, fd, dest_offset,
+			   fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
 }
 
-
-/**
- * stripe_create - If a block-size is specified for the 'name', create the 
- *    file in all the child nodes. If not, create it in only first child.
- *
- * @name- complete path of the file to be created.
- */
 int32_t
-stripe_create (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       int32_t flags,
-	       mode_t mode,
-	       fd_t *fd)
-{
-	stripe_private_t *priv = this->private;
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = NULL;
-
-	/* files created in O_APPEND mode does not allow lseek() on fd */
-	flags &= ~O_APPEND;
-
-	if (priv->first_child_down || priv->nodes_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning EIO");
-		STACK_UNWIND (frame, -1, EIO, fd, loc->inode, NULL);
-		return 0;
-	}
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->stripe_size = stripe_get_matching_bs (loc->path,
-						     priv->pattern,
-						     priv->block_size);
-	frame->local = local;
-	local->inode = loc->inode;
-	loc_copy (&local->loc, loc);
-	local->fd = fd_ref (fd);
-
-	local->call_count = ((stripe_private_t *)this->private)->child_count;
-	
-	trav = this->children;
-	while (trav) {
-		STACK_WIND (frame,
-			    stripe_create_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->create,
-			    loc, flags, mode, fd);
-		trav = trav->next;
-	}
-       
-	return 0;
+stripe_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt    = 0;
+        stripe_local_t *local      = NULL;
+        stripe_local_t *mlocal     = NULL;
+        call_frame_t   *prev       = NULL;
+        call_frame_t   *mframe     = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+        mframe = local->orig_frame;
+        mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+                        mlocal->prebuf_blocks  += prebuf->ia_blocks;
+                        mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+                        correct_file_size(prebuf, mlocal->fctx, prev);
+                        correct_file_size(postbuf, mlocal->fctx, prev);
+
+                        if (mlocal->prebuf_size < prebuf->ia_size)
+                                mlocal->prebuf_size = prebuf->ia_size;
+                        if (mlocal->postbuf_size < postbuf->ia_size)
+                                mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+                /* return the first failure */
+                if (mlocal->op_ret == 0) {
+                        mlocal->op_ret = op_ret;
+                        mlocal->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+                mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+                mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+                mlocal->post_buf.ia_size = mlocal->postbuf_size;
+                mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (zerofill, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+        STRIPE_STACK_DESTROY(frame);
+        return 0;
 }
 
-/**
- * stripe_open_cbk - 
- */
 int32_t
-stripe_open_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 fd_t *fd)
+stripe_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               size_t len, dict_t *xdata)
 {
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-
-		if (op_ret == -1) {
-			local->failed = 1;
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_ret = -1;
-			local->op_errno = op_errno;
-		}
-    
-		if (op_ret >= 0)
-			local->op_ret = op_ret;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed)
-			local->op_ret = -1;
-
-		if (local->op_ret >= 0) {
-			fd_ctx_set (local->fd, this, local->stripe_size);
-		}
-		loc_wipe (&local->loc);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, fd);
-	}
+        stripe_local_t   *local            = NULL;
+        stripe_fd_ctx_t  *fctx             = NULL;
+        int32_t           op_errno         = 1;
+        int32_t           idx              = 0;
+        int32_t           offset_offset    = 0;
+        int32_t           remaining_size   = 0;
+        off_t             fill_size        = 0;
+        uint64_t          stripe_size      = 0;
+        uint64_t          tmp_fctx         = 0;
+        off_t             dest_offset      = 0;
+        call_frame_t     *fframe           = NULL;
+        stripe_local_t   *flocal           = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+        local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+                fframe = copy_frame(frame);
+                flocal = mem_get0(this->local_pool);
+                if (!flocal) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                flocal->orig_frame = frame;
+                fframe->local = flocal;
+
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+                dest_offset = offset + offset_offset;
+                if (fctx->stripe_coalesce)
+                        dest_offset = coalesced_offset(dest_offset,
+                                                       local->stripe_size,
+                                                       fctx->stripe_count);
+
+                STACK_WIND(fframe, stripe_zerofill_cbk, fctx->xl_array[idx],
+                           fctx->xl_array[idx]->fops->zerofill, fd,
+                           dest_offset, fill_size, xdata);
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+        if (fframe)
+                STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
+int32_t
+stripe_release (xlator_t *this, fd_t *fd)
+{
 	return 0;
 }
 
-
-/**
- * stripe_getxattr_cbk - 
- */
-int32_t
-stripe_open_getxattr_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dict_t *dict)
+int
+stripe_forget (xlator_t *this, inode_t *inode)
 {
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
-	xlator_list_t *trav = this->children;
-	stripe_private_t *priv = this->private;
+        uint64_t          tmp_fctx = 0;
+        stripe_fd_ctx_t  *fctx = NULL;
 
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_ret = -1;
-			local->op_errno = op_errno;
-			if (op_errno == ENOTCONN)
-				local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (!local->failed && (local->op_ret != -1)) {
-			/* If getxattr doesn't fails, call open */
-			char size_key[256] = {0,};
-			data_t *stripe_size_data = NULL;
-
-			sprintf (size_key, 
-				 "trusted.%s.stripe-size", this->name);
-			stripe_size_data = dict_get (dict, size_key);
-
-			if (stripe_size_data) {
-				local->stripe_size = 
-					data_to_int64 (stripe_size_data);
-				/*
-				if (local->stripe_size != priv->block_size) {
-					gf_log (this->name, GF_LOG_WARNING,
-						"file(%s) is having different "
-						"block-size", local->loc.path);
-				}
-				*/
-			} else {
-				/* if the file was created using earlier 
-				   versions of stripe */
-				gf_log (this->name, GF_LOG_CRITICAL,
-					"[CRITICAL] Seems like file(%s) "
-					"created using earlier version",
-					local->loc.path);
-			}
-		}
-    
-		local->call_count = priv->child_count;
-
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_open_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->open,
-				    &local->loc, local->flags, local->fd);
-			trav = trav->next;
-		}
-	}
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (inode, err);
 
-	return 0;
-}
+        (void) inode_ctx_del (inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                goto err;
+        }
 
-/**
- * stripe_open - 
- */
-int32_t
-stripe_open (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     int32_t flags,
-	     fd_t *fd)
-{
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-  
-	/* files opened in O_APPEND mode does not allow lseek() on fd */
-	flags &= ~O_APPEND;
-
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning ENOTCONN");
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
-	}
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
 
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->fd = fd;
-	frame->local = local;
-	local->inode = loc->inode;
-	loc_copy (&local->loc, loc);
-
-	/* Striped files */
-	local->flags = flags;
-	local->call_count = priv->child_count;
-	local->stripe_size = stripe_get_matching_bs (loc->path,
-						     priv->pattern,
-						     priv->block_size);
-	
-	if (priv->xattr_supported) {
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_open_getxattr_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->getxattr,
-				    loc, NULL);
-			trav = trav->next;
-		}
-	} else {
-		while (trav) {
-			STACK_WIND (frame,
-				    stripe_open_cbk,
-				    trav->xlator,
-				    trav->xlator->fops->open,
-				    &local->loc, local->flags, local->fd);
-			trav = trav->next;
-		}
-	}
+        if (!fctx->static_array)
+                GF_FREE (fctx->xl_array);
 
-	return 0;
+        GF_FREE (fctx);
+err:
+        return 0;
 }
 
-/**
- * stripe_opendir_cbk - 
- */
 int32_t
-stripe_opendir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    fd_t *fd)
+notify (xlator_t *this, int32_t event, void *data, ...)
 {
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
+        stripe_private_t *priv = NULL;
+        int               down_client = 0;
+        int               i = 0;
+        gf_boolean_t      heard_from_all_children = _gf_false;
+
+        if (!this)
+                return 0;
+
+        priv = this->private;
+        if (!priv)
+                return 0;
+
+        switch (event)
+        {
+        case GF_EVENT_CHILD_UP:
+        {
+                /* get an index number to set */
+                for (i = 0; i < priv->child_count; i++) {
+                        if (data == priv->xl_array[i])
+                                break;
+                }
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_UP bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
+                }
+
+                LOCK (&priv->lock);
+                {
+                        if (data == FIRST_CHILD (this))
+                                priv->first_child_down = 0;
+                        priv->last_event[i] = event;
+                }
+                UNLOCK (&priv->lock);
+        }
+        break;
+        case GF_EVENT_CHILD_CONNECTING:
+        {
+                // 'CONNECTING' doesn't ensure its CHILD_UP, so do nothing
+                goto out;
+        }
+        case GF_EVENT_CHILD_DOWN:
+        {
+                /* get an index number to set */
+                for (i = 0; i < priv->child_count; i++) {
+                        if (data == priv->xl_array[i])
+                                break;
+                }
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
+                }
+
+                LOCK (&priv->lock);
+                {
+                        if (data == FIRST_CHILD (this))
+                                priv->first_child_down = 1;
+                        priv->last_event[i] = event;
+                }
+                UNLOCK (&priv->lock);
+        }
+        break;
+
+        default:
+        {
+                /* */
+                default_notify (this, event, data);
+                goto out;
+        }
+        break;
+        }
+
+        // Consider child as down if it's last_event is not CHILD_UP
+        for (i = 0, down_client = 0; i < priv->child_count; i++)
+                if (priv->last_event[i] != GF_EVENT_CHILD_UP)
+                        down_client++;
+
+        LOCK (&priv->lock);
+        {
+                priv->nodes_down = down_client;
+        }
+        UNLOCK (&priv->lock);
+
+        heard_from_all_children = _gf_true;
+        for (i = 0; i < priv->child_count; i++)
+                if (!priv->last_event[i])
+                        heard_from_all_children = _gf_false;
+
+        if (heard_from_all_children)
+                default_notify (this, event, data);
+out:
+        return 0;
+}
 
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_ret = -1;
-			local->failed = 1;
-			local->op_errno = op_errno;
-		}
-    
-		if (op_ret >= 0) 
-			local->op_ret = op_ret;
-	}
-	UNLOCK (&frame->lock);
+int
+stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
+                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
+{
+        int ret = -1;
+        int call_cnt = 0;
+        stripe_local_t *local = NULL;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
+                return ret;
+        }
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                call_cnt = --local->wind_count;
+
+                /**
+                 * We overwrite ->op_* values here for subsequent faliure
+                 * conditions, hence we propogate the last errno down the
+                 * stack.
+                 */
+                if (op_ret < 0) {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
+                        goto unlock;
+                }
+        }
+
+ unlock:
+        UNLOCK (&frame->lock);
+
+        if (!call_cnt) {
+                STRIPE_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                     local->op_errno, xdata);
+        }
+
+        return 0;
+}
 
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local->fd);
-	}
+#ifdef HAVE_BD_XLATOR
+int
+stripe_is_bd (dict_t *this, char *key, data_t *value, void *data)
+{
+        gf_boolean_t *is_bd = data;
 
-	return 0;
-}
+        if (data == NULL)
+                return 0;
 
+        if (XATTR_IS_BD (key))
+            *is_bd = _gf_true;
 
-/**
- * stripe_opendir - 
- */
-int32_t
-stripe_opendir (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		fd_t *fd)
+        return 0;
+}
+
+inline gf_boolean_t
+stripe_setxattr_is_bd (dict_t *dict)
 {
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
+        gf_boolean_t is_bd = _gf_false;
 
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
+        if (dict == NULL)
+                goto out;
 
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning EIO");
-		STACK_UNWIND (frame, -1, EIO, NULL);
-		return 0;
-	}
+        dict_foreach (dict, stripe_is_bd, &is_bd);
+out:
+        return is_bd;
+}
+#else
+#define stripe_setxattr_is_bd(dict) _gf_false
+#endif
 
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	frame->local = local;
-	local->inode = loc->inode;
-	local->fd = fd;
-	local->call_count = priv->child_count;
-
-	while (trav) {
-		STACK_WIND (frame,
-			    stripe_opendir_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->opendir,
-			    loc, fd);
-		trav = trav->next;
-	}
-  
-	return 0;
+int
+stripe_setxattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
+{
+        int32_t           op_errno = EINVAL;
+        xlator_list_t    *trav     = NULL;
+        stripe_private_t *priv     = NULL;
+        stripe_local_t   *local    = NULL;
+        int               i        = 0;
+        gf_boolean_t      is_bd    = _gf_false;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
+                                   op_errno, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        frame->local = local;
+        local->wind_count = priv->child_count;
+        local->op_ret = local->op_errno = 0;
+
+        is_bd = stripe_setxattr_is_bd (dict);
+
+        /**
+         * Set xattrs for directories on all subvolumes. Additionally
+         * this power is only given to a special client. Bd xlator
+         * also needs xattrs for regular files (ie LVs)
+         */
+        if (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&
+             IA_ISDIR (loc->inode->ia_type)) || is_bd) {
+                for (i = 0; i < priv->child_count; i++, trav = trav->next) {
+                        STACK_WIND (frame, stripe_setxattr_cbk,
+                                    trav->xlator, trav->xlator->fops->setxattr,
+                                    loc, dict, flags, xdata);
+                }
+        } else {
+                local->wind_count = 1;
+                STACK_WIND (frame, stripe_setxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            loc, dict, flags, xdata);
+        }
+
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (setxattr, frame, -1,  op_errno, NULL);
+        return 0;
 }
 
 
-/**
- * stripe_getxattr_cbk - 
- */
-int32_t
-stripe_getxattr_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     dict_t *value)
+int
+stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, value);
-	return 0;
+        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
-/**
- * stripe_getxattr - 
- */
-int32_t
-stripe_getxattr (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 const char *name)
+int
+stripe_is_special_key (dict_t *this,
+                       char *key,
+                       data_t *value,
+                       void *data)
 {
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
+        gf_boolean_t *is_special = NULL;
 
-	STACK_WIND (frame,
-		    stripe_getxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc, name);
+        if (data == NULL) {
+                goto out;
+        }
 
-	return 0;
+        is_special = data;
+
+        if (XATTR_IS_LOCKINFO (key) || XATTR_IS_BD (key))
+                *is_special = _gf_true;
+
+out:
+        return 0;
 }
 
-/**
- * stripe_removexattr - 
- */
 int32_t
-stripe_removexattr (call_frame_t *frame,
-		    xlator_t *this,
-		    loc_t *loc,
-		    const char *name)
+stripe_fsetxattr_everyone_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata)
 {
-	stripe_private_t *priv = this->private;
+        int             call_count = 0;
+        stripe_local_t *local      = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                call_count = --local->wind_count;
+
+                if (op_ret < 0) {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (call_count == 0) {
+                STRIPE_STACK_UNWIND (fsetxattr, frame, local->op_ret,
+                                     local->op_errno, NULL);
+        }
+        return 0;
+}
 
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
+int
+stripe_fsetxattr_to_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              dict_t *dict, int flags, dict_t *xdata)
+{
+        xlator_list_t    *trav  = NULL;
+        stripe_private_t *priv  = NULL;
+        int               ret   = -1;
+        stripe_local_t   *local = NULL;
 
-	if (priv->first_child_down) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"First node down, returning ENOTCONN");
-		STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-		return 0;
-	}
+        priv = this->private;
 
-	STACK_WIND (frame,
-		    stripe_common_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->removexattr,
-		    loc, name);
+        local = mem_get0 (this->local_pool);
+        if (local == NULL) {
+                goto out;
+        }
 
-	return 0;
-}
+        frame->local = local;
 
+        local->wind_count = priv->child_count;
 
-/**
- * stripe_lk_cbk - 
- */
-int32_t
-stripe_lk_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct flock *lock)
-{
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
+        trav = this->children;
 
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-			if (op_errno == ENOTCONN)
-				local->failed = 1;
-		}
-		if (op_ret == 0 && local->op_ret == -1) {
-			/* First successful call, copy the *lock */
-			local->op_ret = 0;
-			local->lock = *lock;
-		}
-	}
-	UNLOCK (&frame->lock);
+        while (trav) {
+                STACK_WIND (frame, stripe_fsetxattr_everyone_cbk,
+                            trav->xlator, trav->xlator->fops->fsetxattr,
+                            fd, dict, flags, xdata);
+                trav = trav->next;
+        }
 
-	if (!callcnt) {
-		if (local->failed)
-			local->op_ret = -1;
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, &local->lock);
-	}
-	return 0;
+        ret = 0;
+out:
+        return ret;
 }
 
+inline gf_boolean_t
+stripe_fsetxattr_is_special (dict_t *dict)
+{
+        gf_boolean_t is_spl = _gf_false;
 
-/**
- * stripe_lk - 
- */
-int32_t
-stripe_lk (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd,
-	   int32_t cmd,
-	   struct flock *lock)
-{
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-	stripe_private_t *priv = this->private;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (fd);
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-  
-	local->call_count = priv->child_count;
-	
-	while (trav) {
-		STACK_WIND (frame,	      
-			    stripe_lk_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->lk,
-			    fd, cmd, lock);
-		trav = trav->next;
-	}
+        if (dict == NULL) {
+                goto out;
+        }
 
-	return 0;
+        dict_foreach (dict, stripe_is_special_key, &is_spl);
+
+out:
+        return is_spl;
 }
 
-/**
- * stripe_writedir - 
- */
-int32_t
-stripe_setdents (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 int32_t flags,
-		 dir_entry_t *entries,
-		 int32_t count)
-{
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (fd);
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-	local->call_count = priv->child_count;
-
-	while (trav) {
-		STACK_WIND (frame,	      
-			    stripe_stack_unwind_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->setdents,
-			    fd, flags, entries, count);
-		trav = trav->next;
-	}
+int
+stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  dict_t *dict, int flags, dict_t *xdata)
+{
+        int32_t      op_ret = -1, ret = -1, op_errno = EINVAL;
+        gf_boolean_t is_spl = _gf_false;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
+                                   op_errno, err);
+
+        is_spl = stripe_fsetxattr_is_special (dict);
+        if (is_spl) {
+                ret = stripe_fsetxattr_to_everyone (frame, this, fd, dict,
+                                                    flags, xdata);
+                if (ret < 0) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+
+                goto out;
+        }
+
+        STACK_WIND (frame, stripe_fsetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+out:
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
+        return 0;
+}
 
-	return 0;
+int
+stripe_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        STRIPE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
+int
+stripe_removexattr (call_frame_t *frame, xlator_t *this,
+                    loc_t *loc, const char *name, dict_t *xdata)
+{
+        int32_t         op_errno = EINVAL;
 
-/**
- * stripe_flush - 
- */
-int32_t
-stripe_flush (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd)
-{
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (fd);
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-	local->call_count = priv->child_count;
-	
-	while (trav) {
-		STACK_WIND (frame,	      
-			    stripe_stack_unwind_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->flush,
-			    fd);
-		trav = trav->next;
-	}
+        VALIDATE_OR_GOTO (this, err);
 
-	return 0;
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
+                                 name, op_errno, err);
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        STACK_WIND (frame, stripe_removexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
+        return 0;
 }
 
 
-/**
- * stripe_close - 
- */
-int32_t
-stripe_release (xlator_t *this,
-		fd_t *fd)
+int
+stripe_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	return 0;
+        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
+int
+stripe_fremovexattr (call_frame_t *frame, xlator_t *this,
+                     fd_t *fd, const char *name, dict_t *xdata)
+{
+        int32_t         op_ret   = -1;
+        int32_t         op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
+                                 name, op_errno, err);
+
+        STACK_WIND (frame, stripe_fremovexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+ err:
+        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
 
-/**
- * stripe_fsync - 
- */
 int32_t
-stripe_fsync (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      int32_t flags)
-{
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (fd);
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-	local->call_count = priv->child_count;
-	
-	while (trav) {
-		STACK_WIND (frame,	      
-			    stripe_stack_unwind_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->fsync,
-			    fd, flags);
-		trav = trav->next;
-	}
-
-	return 0;
+stripe_readdirp_lookup_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int op_ret, int op_errno,
+                            inode_t *inode, struct iatt *stbuf,
+                            dict_t *xattr, struct iatt *parent)
+{
+        stripe_local_t          *local          = NULL;
+        call_frame_t            *main_frame     = NULL;
+        stripe_local_t          *main_local     = NULL;
+        gf_dirent_t             *entry          = NULL;
+        call_frame_t            *prev           = NULL;
+        int                      done           = 0;
+
+        local = frame->local;
+        prev = cookie;
+
+        entry = local->dirent;
+
+        main_frame = local->orig_frame;
+        main_local = main_frame->local;
+        LOCK (&frame->lock);
+        {
+
+                local->call_count--;
+                if (!local->call_count)
+                        done = 1;
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret = op_ret;
+                        goto unlock;
+                }
+
+		if (stripe_ctx_handle(this, prev, local, xattr))
+			gf_log(this->name, GF_LOG_ERROR,
+				"Error getting fctx info from dict.");
+
+		correct_file_size(stbuf, local->fctx, prev);
+
+                stripe_iatt_merge (stbuf, &entry->d_stat);
+                local->stbuf_blocks += stbuf->ia_blocks;
+        }
+unlock:
+        UNLOCK(&frame->lock);
+
+        if (done) {
+                inode_ctx_put (entry->inode, this,
+                               (uint64_t) (long)local->fctx);
+
+                done = 0;
+                LOCK (&main_frame->lock);
+                {
+                        main_local->wind_count--;
+                        if (!main_local->wind_count)
+                                done = 1;
+                        if (local->op_ret == -1) {
+                                main_local->op_errno = local->op_errno;
+                                main_local->op_ret = local->op_ret;
+                        }
+                        entry->d_stat.ia_blocks = local->stbuf_blocks;
+                }
+                UNLOCK (&main_frame->lock);
+                if (done) {
+                        main_frame->local = NULL;
+                        STRIPE_STACK_UNWIND (readdir, main_frame,
+                                             main_local->op_ret,
+                                             main_local->op_errno,
+                                             &main_local->entries, NULL);
+                        gf_dirent_free (&main_local->entries);
+                        stripe_local_wipe (main_local);
+                        mem_put (main_local);
+                }
+                frame->local = NULL;
+                stripe_local_wipe (local);
+                mem_put (local);
+                STRIPE_STACK_DESTROY (frame);
+        }
+
+        return 0;
 }
 
-
-/**
- * stripe_fstat - 
- */
 int32_t
-stripe_fstat (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd)
-{
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (fd);
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-	local->inode = fd->inode;
-	local->call_count = priv->child_count;
-	
-	while (trav) {
-		STACK_WIND (frame,	      
-			    stripe_stack_unwind_buf_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->fstat,
-			    fd);
-		trav = trav->next;
-	}
+stripe_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     gf_dirent_t *orig_entries, dict_t *xdata)
+{
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+        gf_dirent_t    *local_entry = NULL;
+        gf_dirent_t    *tmp_entry = NULL;
+        xlator_list_t  *trav = NULL;
+        loc_t          loc = {0, };
+        int32_t        count = 0;
+        stripe_private_t *priv = NULL;
+        int32_t        subvols = 0;
+        dict_t         *xattrs = NULL;
+        call_frame_t   *local_frame = NULL;
+        stripe_local_t *local_ent = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+        prev  = cookie;
+        local = frame->local;
+        trav = this->children;
+        priv = this->private;
+
+        subvols = priv->child_count;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        local->op_ret = op_ret;
+                        goto unlock;
+                } else {
+                        local->op_ret = op_ret;
+                        list_splice_init (&orig_entries->list,
+                                          &local->entries.list);
+                        local->wind_count = op_ret;
+                }
+
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (op_ret == -1)
+                goto out;
+
+        xattrs = dict_new ();
+        if (xattrs)
+                (void) stripe_xattr_request_build (this, xattrs, 0, 0, 0, 0);
+        count = op_ret;
+        list_for_each_entry_safe (local_entry, tmp_entry,
+                                  (&local->entries.list), list) {
+
+                if (!local_entry)
+                        break;
+                if (!IA_ISREG (local_entry->d_stat.ia_type)) {
+                        LOCK (&frame->lock);
+                        {
+                                local->wind_count--;
+                                count = local->wind_count;
+                        }
+                        UNLOCK (&frame->lock);
+                        continue;
+                }
+
+                local_frame = copy_frame (frame);
+
+                if (!local_frame) {
+                        op_errno = ENOMEM;
+                        op_ret = -1;
+                        goto out;
+                }
+
+                local_ent = mem_get0 (this->local_pool);
+                if (!local_ent) {
+                        op_errno = ENOMEM;
+                        op_ret = -1;
+                        goto out;
+                }
+
+                loc.inode = inode_ref (local_entry->inode);
+
+                uuid_copy (loc.gfid, local_entry->d_stat.ia_gfid);
+
+                local_ent->orig_frame = frame;
+
+                local_ent->call_count = subvols;
+
+                local_ent->dirent = local_entry;
+
+                local_frame->local = local_ent;
+
+                trav = this->children;
+                while (trav) {
+                        STACK_WIND (local_frame, stripe_readdirp_lookup_cbk,
+                                    trav->xlator, trav->xlator->fops->lookup,
+                                    &loc, xattrs);
+                        trav = trav->next;
+                }
+                loc_wipe (&loc);
+        }
+out:
+        if (!count) {
+                /* all entries are directories */
+                frame->local = NULL;
+                STRIPE_STACK_UNWIND (readdir, frame, local->op_ret,
+                                     local->op_errno, &local->entries, NULL);
+                gf_dirent_free (&local->entries);
+                stripe_local_wipe (local);
+                mem_put (local);
+        }
+        if (xattrs)
+                dict_unref (xattrs);
+        return 0;
 
-	return 0;
 }
+int32_t
+stripe_readdirp (call_frame_t *frame, xlator_t *this,
+                 fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        stripe_local_t  *local  = NULL;
+        stripe_private_t *priv = NULL;
+        xlator_list_t   *trav = NULL;
+        int             op_errno = -1;
 
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
 
-/**
- * stripe_fchmod - 
- */
-int32_t 
-stripe_fchmod (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       mode_t mode)
-{
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (fd);
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-	local->inode = fd->inode;
-	local->call_count = priv->child_count;
-	
-	while (trav) {
-		STACK_WIND (frame,	      
-			    stripe_stack_unwind_buf_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->fchmod,
-			    fd, mode);
-		trav = trav->next;
-	}
+        priv = this->private;
+        trav = this->children;
 
-	return 0;
-}
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
 
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-/**
- * stripe_fchown - 
- */
-int32_t 
-stripe_fchown (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       uid_t uid,
-	       gid_t gid)
-{
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (fd);
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-	local->inode = fd->inode;
-	local->call_count = priv->child_count;
-	
-	while (trav) {
-		STACK_WIND (frame,	      
-			    stripe_stack_unwind_buf_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->fchown,
-			    fd, uid, gid);
-		trav = trav->next;
-	}
+        frame->local = local;
 
-	return 0;
-}
+        local->fd = fd_ref (fd);
 
+        local->wind_count = 0;
 
-/**
- * stripe_ftruncate - 
- */
-int32_t
-stripe_ftruncate (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  off_t offset)
-{
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (fd);
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-	local->inode = fd->inode;
-	local->call_count = priv->child_count;
-	
-	while (trav) {
-		STACK_WIND (frame,	      
-			    stripe_stack_unwind_buf_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->ftruncate,
-			    fd, offset);
-		trav = trav->next;
-	}
+        local->count = 0;
+        local->op_ret = -1;
+        INIT_LIST_HEAD(&local->entries);
 
-	return 0;
-}
+        if (!trav)
+                goto err;
 
+        STACK_WIND (frame, stripe_readdirp_cbk, trav->xlator,
+                    trav->xlator->fops->readdirp, fd, size, off, xdata);
+        return 0;
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        STRIPE_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+
+}
 
-/**
- * stripe_releasedir - 
- */
 int32_t
-stripe_releasedir (xlator_t *this,
-		   fd_t *fd)
+mem_acct_init (xlator_t *this)
 {
-	return 0;
+        int     ret = -1;
+
+        if (!this)
+                goto out;
+
+        ret = xlator_mem_acct_init (this, gf_stripe_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        "failed");
+                goto out;
+        }
+
+out:
+        return ret;
 }
 
+static int
+clear_pattern_list (stripe_private_t *priv)
+{
+        struct stripe_options *prev = NULL;
+        struct stripe_options *trav = NULL;
+        int                    ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
+
+        trav = priv->pattern;
+        priv->pattern = NULL;
+        while (trav) {
+                prev = trav;
+                trav = trav->next;
+                GF_FREE (prev);
+        }
+
+        ret = 0;
+ out:
+        return ret;
 
-/**
- * stripe_fsyncdir - 
- */
-int32_t
-stripe_fsyncdir (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 int32_t flags)
-{
-	stripe_local_t *local = NULL;
-	stripe_private_t *priv = this->private;
-	xlator_list_t *trav = this->children;
-
-	STRIPE_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (fd);
-
-	/* Initialization */
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->op_ret = -1;
-	frame->local = local;
-	local->call_count = priv->child_count;
-
-	while (trav) {
-		STACK_WIND (frame,	      
-			    stripe_stack_unwind_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->fsyncdir,
-			    fd,
-			    flags);
-		trav = trav->next;
-	}
 
-	return 0;
 }
 
 
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+
+        stripe_private_t *priv = NULL;
+        data_t           *data = NULL;
+        int               ret = -1;
+        volume_option_t  *opt = NULL;
+
+        GF_ASSERT (this);
+        GF_ASSERT (this->private);
+
+        priv = this->private;
+
+
+        ret = 0;
+        LOCK (&priv->lock);
+        {
+                ret = clear_pattern_list (priv);
+                if (ret)
+                         goto unlock;
+
+                data = dict_get (options, "block-size");
+                if (data) {
+                        ret = set_stripe_block_size (this, priv, data->data);
+                        if (ret)
+                                goto unlock;
+                } else {
+                        opt = xlator_volume_option_get (this, "block-size");
+                        if (!opt) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "option 'block-size' not found");
+                                ret = -1;
+                                goto unlock;
+                        }
+
+                        if (gf_string2bytesize (opt->default_value, &priv->block_size)){
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to set default block-size ");
+                                ret = -1;
+                                goto unlock;
+                        }
+                }
+
+		GF_OPTION_RECONF("coalesce", priv->coalesce, options, bool,
+				unlock);
+        }
+ unlock:
+        UNLOCK (&priv->lock);
+        if (ret)
+                goto out;
+
+        ret = 0;
+ out:
+        return ret;
+
+}
+
 /**
- * stripe_single_readv_cbk - This function is used as return fn, when the 
- *     file name doesn't match the pattern specified for striping.
+ * init - This function is called when xlator-graph gets initialized.
+ *     The option given in volfiles are parsed here.
+ * @this -
  */
 int32_t
-stripe_single_readv_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 struct iovec *vector,
-			 int32_t count,
-			 struct stat *stbuf)
+init (xlator_t *this)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf);
-	return 0;
+        stripe_private_t *priv = NULL;
+        volume_option_t  *opt = NULL;
+        xlator_list_t    *trav = NULL;
+        data_t           *data = NULL;
+        int32_t           count = 0;
+        int               ret = -1;
+
+        if (!this)
+                goto out;
+
+        trav = this->children;
+        while (trav) {
+                count++;
+                trav = trav->next;
+        }
+
+        if (!count) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stripe configured without \"subvolumes\" option. "
+                        "exiting");
+                goto out;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+        if (count == 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stripe configured with only one \"subvolumes\" option."
+                        " please check the volume. exiting");
+                goto out;
+        }
+
+        priv = GF_CALLOC (1, sizeof (stripe_private_t),
+                          gf_stripe_mt_stripe_private_t);
+
+        if (!priv)
+                goto out;
+        priv->xl_array = GF_CALLOC (count, sizeof (xlator_t *),
+                                    gf_stripe_mt_xlator_t);
+        if (!priv->xl_array)
+                goto out;
+
+        priv->last_event = GF_CALLOC (count, sizeof (int),
+                                      gf_stripe_mt_int32_t);
+        if (!priv->last_event)
+                goto out;
+
+        priv->child_count = count;
+        LOCK_INIT (&priv->lock);
+
+        trav = this->children;
+        count = 0;
+        while (trav) {
+                priv->xl_array[count++] = trav->xlator;
+                trav = trav->next;
+        }
+
+        if (count > 256) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "maximum number of stripe subvolumes supported "
+                        "is 256");
+                goto out;
+        }
+
+        ret = 0;
+        LOCK (&priv->lock);
+        {
+                opt = xlator_volume_option_get (this, "block-size");
+                if (!opt) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "option 'block-size' not found");
+                        ret = -1;
+                        goto unlock;
+                }
+                if (gf_string2bytesize (opt->default_value, &priv->block_size)){
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set default block-size ");
+                        ret = -1;
+                        goto unlock;
+                }
+                /* option stripe-pattern *avi:1GB,*pdf:16K */
+                data = dict_get (this->options, "block-size");
+                if (data) {
+                        ret = set_stripe_block_size (this, priv, data->data);
+                        if (ret)
+                                goto unlock;
+                }
+        }
+ unlock:
+        UNLOCK (&priv->lock);
+        if (ret)
+                goto out;
+
+        GF_OPTION_INIT ("use-xattr", priv->xattr_supported, bool, out);
+        /* notify related */
+        priv->nodes_down = priv->child_count;
+
+	GF_OPTION_INIT("coalesce", priv->coalesce, bool, out);
+
+        this->local_pool = mem_pool_new (stripe_local_t, 128);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
+        this->private = priv;
+
+        ret = 0;
+out:
+        if (ret) {
+                if (priv) {
+                        GF_FREE (priv->xl_array);
+                        GF_FREE (priv);
+                }
+        }
+        return ret;
 }
 
 /**
- * stripe_readv_cbk - get all the striped reads, and order it properly, send it
- *        to above layer after putting it in a single vector.
+ * fini -   Free all the private variables
+ * @this -
  */
+void
+fini (xlator_t *this)
+{
+        stripe_private_t      *priv = NULL;
+        struct stripe_options *prev = NULL;
+        struct stripe_options *trav = NULL;
+
+        if (!this)
+                goto out;
+
+        priv = this->private;
+        if (priv) {
+                this->private = NULL;
+                GF_FREE (priv->xl_array);
+
+                trav = priv->pattern;
+                while (trav) {
+                        prev = trav;
+                        trav = trav->next;
+                        GF_FREE (prev);
+                }
+                GF_FREE (priv->last_event);
+                LOCK_DESTROY (&priv->lock);
+                GF_FREE (priv);
+        }
+
+out:
+        return;
+}
+
 int32_t
-stripe_readv_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iovec *vector,
-		  int32_t count,
-		  struct stat *stbuf)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	call_frame_t *main_frame = NULL;
-	stripe_local_t *main_local = NULL;
-	stripe_local_t *local = frame->local;
-
-	index = local->node_index;
-	main_frame = local->orig_frame;
-	main_local = main_frame->local;
-
-	LOCK (&main_frame->lock);
-	{
-		main_local->replies[index].op_ret = op_ret;
-		main_local->replies[index].op_errno = op_errno;
-		if (op_ret >= 0) {
-			main_local->replies[index].stbuf  = *stbuf;
-			main_local->replies[index].count  = count;
-			main_local->replies[index].vector = 
-				iov_dup (vector, count);
-
-			if (frame->root->rsp_refs)
-				dict_copy (frame->root->rsp_refs, 
-					   main_frame->root->rsp_refs);
-		}
-		callcnt = ++main_local->call_count;
-	}
-	UNLOCK(&main_frame->lock);
-
-	if (callcnt == main_local->wind_count) {
-		int32_t final_count = 0;
-		struct iovec *final_vec = NULL;
-		struct stat tmp_stbuf = {0,};
-		dict_t *refs = main_frame->root->rsp_refs;
-
-		op_ret = 0;
-		memcpy (&tmp_stbuf, &main_local->replies[0].stbuf, 
-			sizeof (struct stat));
-		for (index=0; index < main_local->wind_count; index++) {
-			/* TODO: check whether each stripe returned 'expected'
-			 * number of bytes 
-			 */
-			if (main_local->replies[index].op_ret == -1) {
-				op_ret = -1;
-				op_errno = main_local->replies[index].op_errno;
-				break;
-			}
-			op_ret += main_local->replies[index].op_ret;
-			final_count += main_local->replies[index].count;
-			/* TODO: Do I need to send anything more in stbuf? */
-			if (tmp_stbuf.st_size < 
-			    main_local->replies[index].stbuf.st_size) {
-				tmp_stbuf.st_size = 
-					main_local->replies[index].stbuf.st_size;
-			}
-		}
-		if (op_ret != -1) {
-			final_vec = CALLOC (final_count, 
-					    sizeof (struct iovec));
-			ERR_ABORT (final_vec);
-			final_count = 0;
-
-			for (index=0; 
-			     index < main_local->wind_count; index++) {
-				memcpy (final_vec + final_count,
-					main_local->replies[index].vector,
-					(main_local->replies[index].count * 
-					 sizeof (struct iovec)));
-				final_count += 
-					main_local->replies[index].count;
-
-				free (main_local->replies[index].vector);
-			}
-		} else {
-			final_vec = NULL;
-			final_count = 0;
-		}
-		/* */
-		FREE (main_local->replies);
-		refs = main_frame->root->rsp_refs;
-		STACK_UNWIND (main_frame, op_ret, op_errno, 
-			      final_vec, final_count, &tmp_stbuf);
-
-		dict_unref (refs);
-		if (final_vec)
-			free (final_vec);
-	}
+stripe_getxattr_unwind (call_frame_t *frame,
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
-	STACK_DESTROY (frame->root);
-	return 0;
+{
+        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
-/**
- * stripe_readv - 
- */
-int32_t
-stripe_readv (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      size_t size,
-	      off_t offset)
-{
-	int32_t index = 0;
-	int32_t num_stripe = 0;
-	size_t frame_size = 0;
-	off_t rounded_end = 0;
-	uint64_t stripe_size = 0;
-	off_t rounded_start = 0;
-	off_t frame_offset = offset;
-	stripe_local_t *local = NULL;
-	call_frame_t *rframe = NULL;
-	stripe_local_t *rlocal = NULL;
-	xlator_list_t *trav = this->children;
-	stripe_private_t *priv = this->private;
-
-	fd_ctx_get (fd, this, &stripe_size);
-	if (!stripe_size) {
-		STACK_UNWIND (frame, -1, EINVAL, NULL, 0, NULL);
-		return 0;
-	}
+int
+stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, dict_t *xattr,
+                              dict_t *xdata)
+{
 
-	/* The file is stripe across the child nodes. Send the read request 
-	 * to the child nodes appropriately after checking which region of 
-	 * the file is in which child node. Always '0-<stripe_size>' part of
-	 * the file resides in the first child.
-	 */
-	rounded_start = floor (offset, stripe_size);
-	rounded_end = roof (offset+size, stripe_size);
-	num_stripe = (rounded_end - rounded_start) / stripe_size;
-	
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	local->wind_count = num_stripe;
-	frame->local = local;
-	frame->root->rsp_refs = dict_ref (get_new_dict ());
-	
-	/* This is where all the vectors should be copied. */
-	local->replies = CALLOC (1, num_stripe * 
-				 sizeof (struct readv_replies));
-	ERR_ABORT (local->replies);
-	
-	for (index = 0;
-	     index < ((offset / stripe_size) % priv->child_count);
-	     index++) {
-		trav = trav->next;
-	}
-    
-	for (index = 0; index < num_stripe; index++) {
-		rframe = copy_frame (frame);
-		rlocal = CALLOC (1, sizeof (stripe_local_t));
-		ERR_ABORT (rlocal);
-		
-		frame_size = min (roof (frame_offset+1, stripe_size),
-				  (offset + size)) - frame_offset;
-		
-		rlocal->node_index = index;
-		rlocal->orig_frame = frame;
-		rframe->local = rlocal;
-		STACK_WIND (rframe,
-			    stripe_readv_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->readv,
-			    fd, frame_size, frame_offset);
-      
-		frame_offset += frame_size;
-
-		trav = trav->next ? trav->next : this->children;
-	}
+        char        size_key[256]  = {0,};
+        char        index_key[256] = {0,};
+        char        count_key[256] = {0,};
+        char        coalesce_key[256] = {0,};
 
-	return 0;
-}
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
 
+        if (!xattr || (op_ret == -1))
+            goto out;
 
-/**
- * stripe_writev_cbk - 
- */
-int32_t
-stripe_writev_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct stat *stbuf)
-{
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
-	LOCK(&frame->lock);
-	{
-		callcnt = ++local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-			local->op_ret = -1;
-		}
-		if (op_ret >= 0) {
-			local->op_ret += op_ret;
-			local->stbuf = *stbuf;
-		}
-	}
-	UNLOCK (&frame->lock);
+        sprintf (size_key, "trusted.%s.stripe-size", this->name);
+        sprintf (count_key, "trusted.%s.stripe-count", this->name);
+        sprintf (index_key, "trusted.%s.stripe-index", this->name);
+	sprintf (coalesce_key, "trusted.%s.stripe-coalesce", this->name);
 
-	if ((callcnt == local->wind_count) && local->unwind) {
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, &local->stbuf);
-	}
-	return 0;
-}
+        dict_del (xattr, size_key);
+        dict_del (xattr, count_key);
+        dict_del (xattr, index_key);
+        dict_del (xattr, coalesce_key);
 
+out:
+        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+
+        return 0;
 
-/**
- * stripe_single_writev_cbk - 
- */
-int32_t
-stripe_single_writev_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  struct stat *stbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, stbuf);
-	return 0;
 }
-/**
- * stripe_writev - 
- */
-int32_t
-stripe_writev (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       struct iovec *vector,
-	       int32_t count,
-	       off_t offset)
-{
-	int32_t idx = 0;
-	int32_t total_size = 0;
-	int32_t offset_offset = 0;
-	int32_t remaining_size = 0;
-	int32_t tmp_count = count;
-	off_t fill_size = 0;
-	uint64_t stripe_size = 0;
-	struct iovec *tmp_vec = vector;
-	stripe_private_t *priv = this->private;
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = NULL;
-
-	fd_ctx_get (fd, this, &stripe_size);
-	if (!stripe_size) {
-		STACK_UNWIND (frame, -1, EINVAL, NULL);
-		return 0;
-	}
 
-	/* File has to be stripped across the child nodes */
-	for (idx = 0; idx< count; idx ++) {
-		total_size += tmp_vec[idx].iov_len;
-	}
-	remaining_size = total_size;
-
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	frame->local = local;
-	local->stripe_size = stripe_size;
-
-	while (1) {
-		/* Send striped chunk of the vector to child 
-		   nodes appropriately. */
-		trav = this->children;
-		
-		idx = (((offset + offset_offset) / 
-			local->stripe_size) % priv->child_count);
-		while (idx) {
-			trav = trav->next;
-			idx--;
-		}
-		fill_size = (local->stripe_size - 
-			     ((offset + offset_offset) % local->stripe_size));
-		if (fill_size > remaining_size)
-			fill_size = remaining_size;
-
-		remaining_size -= fill_size;
-
-		tmp_count = iov_subset (vector, count, offset_offset,
-					offset_offset + fill_size, NULL);
-		tmp_vec = CALLOC (tmp_count, sizeof (struct iovec));
-		ERR_ABORT (tmp_vec);
-		tmp_count = iov_subset (vector, count, offset_offset,
-					offset_offset + fill_size, tmp_vec);
-		
-		local->wind_count++;
-		if (remaining_size == 0)
-			local->unwind = 1;
-
-		STACK_WIND(frame,
-			   stripe_writev_cbk,
-			   trav->xlator,
-			   trav->xlator->fops->writev,
-			   fd, tmp_vec, tmp_count, offset + offset_offset);
-		FREE (tmp_vec);
-		offset_offset += fill_size;
-		if (remaining_size == 0)
-			break;
-	}
+int
+stripe_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        int                     call_cnt = 0;
+        stripe_local_t         *local = NULL;
 
-	return 0;
-}
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
 
+        local = frame->local;
 
+        LOCK (&frame->lock);
+        {
+                call_cnt = --local->wind_count;
+        }
+        UNLOCK (&frame->lock);
 
-/* Management operations */
+        if (!xattr || (op_ret < 0))
+                goto out;
 
-/**
- * stripe_stats_cbk - Add all the fields received from different clients. 
- *    Once all the clients return, send stats to above layer.
- * 
- */
-int32_t
-stripe_stats_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct xlator_stats *stats)
-{
-	int32_t callcnt = 0;
-	stripe_local_t *local = frame->local;
-
-	LOCK(&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_WARNING, 
-				"%s returned error %s",
-				((call_frame_t *)cookie)->this->name, 
-				strerror (op_errno));
-			local->op_ret = -1;
-			local->op_errno = op_errno;
-		}
-		if (op_ret == 0) {
-			if (local->op_ret == -2) {
-				/* This is to make sure this is the 
-				   first time */
-				local->stats = *stats;
-				local->op_ret = 0;
-			} else {
-				local->stats.nr_files += stats->nr_files;
-				local->stats.free_disk += stats->free_disk;
-				local->stats.disk_usage += stats->disk_usage;
-				local->stats.nr_clients += stats->nr_clients;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
+        local->op_ret = 0;
 
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			      &local->stats);
-	}
+        if (!local->xattr) {
+                local->xattr = dict_ref (xattr);
+        } else {
+                stripe_aggregate_xattr (local->xattr, xattr);
+        }
 
-	return 0;
+out:
+        if (!call_cnt) {
+                STRIPE_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+                                     local->xattr, xdata);
+        }
+
+        return 0;
 }
 
-/**
- * stripe_stats - 
- */
 int32_t
-stripe_stats (call_frame_t *frame,
-	      xlator_t *this,
-	      int32_t flags)
-{
-	stripe_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-
-	local = CALLOC (1, sizeof (stripe_local_t));
-	ERR_ABORT (local);
-	frame->local = local;
-	local->op_ret = -2; /* to be used as a flag in _cbk */
-	local->call_count = ((stripe_private_t*)this->private)->child_count;
-	while (trav) {
-		STACK_WIND (frame,
-			    stripe_stats_cbk,
-			    trav->xlator,
-			    trav->xlator->mops->stats,
-			    flags);
-		trav = trav->next;
-	}
-	return 0;
+stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int32_t op_ret, int32_t op_errno,
+                      dict_t *dict, dict_t *xdata)
+{
+        stripe_local_t      *local         = NULL;
+        int32_t              callcnt       = 0;
+        int32_t              ret           = -1;
+        long                 cky           = 0;
+        void                *xattr_val     = NULL;
+        void                *xattr_serz    = NULL;
+        stripe_xattr_sort_t *xattr         = NULL;
+        dict_t              *stripe_xattr  = NULL;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
+                return ret;
+        }
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        if (local->xsel[0] == '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "Empty xattr in cbk");
+                return ret;
+        }
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->wind_count;
+
+                if (!dict || (op_ret < 0))
+                        goto out;
+
+                if (!local->xattr_list)
+                        local->xattr_list = (stripe_xattr_sort_t *)
+                                GF_CALLOC (local->nallocs,
+                                           sizeof (stripe_xattr_sort_t),
+                                           gf_stripe_mt_xattr_sort_t);
+
+                if (local->xattr_list) {
+                        xattr = local->xattr_list + (int32_t) cky;
+
+                        ret = dict_get_ptr_and_len (dict, local->xsel,
+                                                    &xattr_val,
+                                                    &xattr->xattr_len);
+                        if (xattr->xattr_len == 0)
+                                goto out;
+
+                        xattr->pos = cky;
+                        xattr->xattr_value = gf_memdup (xattr_val,
+                                                        xattr->xattr_len);
+
+                        if (xattr->xattr_value != NULL)
+                                local->xattr_total_len += xattr->xattr_len + 1;
+                }
+        }
+ out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->xattr_total_len)
+                        goto unwind;
+
+                stripe_xattr = dict_new ();
+                if (!stripe_xattr)
+                        goto unwind;
+
+                /* select filler based on ->xsel */
+                if (XATTR_IS_PATHINFO (local->xsel))
+                        ret = stripe_fill_pathinfo_xattr (this, local,
+                                                          (char **)&xattr_serz);
+                else if (XATTR_IS_LOCKINFO (local->xsel)) {
+                        ret = stripe_fill_lockinfo_xattr (this, local,
+                                                          &xattr_serz);
+                } else {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Unknown xattr in xattr request");
+                        goto unwind;
+                }
+
+                if (!ret) {
+                        ret = dict_set_dynptr (stripe_xattr, local->xsel,
+                                               xattr_serz,
+                                               local->xattr_total_len);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Can't set %s key in dict",
+                                        local->xsel);
+                }
+
+        unwind:
+                STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
+                                     stripe_xattr, NULL);
+
+                ret = stripe_free_xattr_str (local);
+
+                GF_FREE (local->xattr_list);
+
+                if (stripe_xattr)
+                        dict_unref (stripe_xattr);
+        }
+
+        return ret;
 }
 
-/**
- * notify
- */
 int32_t
-notify (xlator_t *this,
-        int32_t event,
-        void *data,
-        ...)
-{
-	stripe_private_t *priv = this->private;
-	int down_client = 0;
-	int i = 0;
-
-	if (!priv)
-		return 0;
-
-	switch (event) 
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		/* get an index number to set */
-		for (i = 0; i < priv->child_count; i++) {
-			if (data == priv->xl_array[i])
-				break;
-		}
-		priv->state[i] = 1;
-		for (i = 0; i < priv->child_count; i++) {
-			if (!priv->state[i])
-				down_client++;
-		}
-
-		LOCK (&priv->lock);
-		{
-			priv->nodes_down = down_client;
+stripe_getxattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, const char *name, dict_t *xdata)
+{
+        stripe_local_t    *local    = NULL;
+        xlator_list_t     *trav     = NULL;
+        stripe_private_t  *priv     = NULL;
+        int32_t            op_errno = EINVAL;
+        int                i        = 0;
+        xlator_t         **sub_volumes;
+        int                ret      = 0;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        loc_copy (&local->loc, loc);
+
+
+        if (name && (strcmp (GF_XATTR_MARKER_KEY, name) == 0)
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                local->marker.call_count = priv->child_count;
+
+                sub_volumes = alloca ( priv->child_count *
+                                       sizeof (xlator_t *));
+                for (i = 0, trav = this->children; trav ;
+                     trav = trav->next, i++) {
+
+                        *(sub_volumes + i)  = trav->xlator;
+
+                }
+
+                if (cluster_getmarkerattr (frame, this, loc, name,
+                                           local, stripe_getxattr_unwind,
+                                           sub_volumes, priv->child_count,
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
+                return 0;
+        }
+
+        if (name && strncmp (name, GF_XATTR_QUOTA_SIZE_KEY,
+                             strlen (GF_XATTR_QUOTA_SIZE_KEY)) == 0) {
+                local->wind_count = priv->child_count;
+
+                for (i = 0, trav=this->children; i < priv->child_count; i++,
+                             trav = trav->next) {
+                        STACK_WIND (frame, stripe_getxattr_cbk,
+                                    trav->xlator, trav->xlator->fops->getxattr,
+                                    loc, name, xdata);
+                }
+
+                return 0;
+        }
+
+        if (name &&
+            ((strncmp (name, GF_XATTR_PATHINFO_KEY,
+                       strlen (GF_XATTR_PATHINFO_KEY)) == 0))) {
+                if (IA_ISREG (loc->inode->ia_type)) {
+                        ret = inode_ctx_get (loc->inode, this,
+                                             (uint64_t *) &local->fctx);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "stripe size unavailable from fctx"
+                                        " relying on pathinfo could lead to"
+                                        " wrong results");
+                }
+
+                local->nallocs = local->wind_count = priv->child_count;
+                (void) strncpy (local->xsel, name, strlen (name));
+
+                /**
+                 * for xattrs that need info from all childs, fill ->xsel
+                 * as above and call the filler function in cbk based on
+                 * it
+                 */
+                for (i = 0, trav = this->children; i < priv->child_count; i++,
+                     trav = trav->next) {
+                        STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
+                                           (void *) (long) i, trav->xlator,
+                                           trav->xlator->fops->getxattr,
+                                           loc, name, xdata);
+                }
+
+                return 0;
+        }
+
+        if (name &&(*priv->vol_uuid)) {
+                if ((match_uuid_local (name, priv->vol_uuid) == 0)
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+
+                        if (!IA_FILE_OR_DIR (loc->inode->ia_type))
+                                local->marker.call_count = 1;
+                        else
+                                local->marker.call_count = priv->child_count;
+
+                        sub_volumes = alloca (local->marker.call_count *
+                                              sizeof (xlator_t *));
+
+                        for (i = 0, trav = this->children;
+                             i < local->marker.call_count;
+                             i++, trav = trav->next) {
+                                *(sub_volumes + i) = trav->xlator;
+
+                        }
+
+                        if (cluster_getmarkerattr (frame, this, loc, name,
+                                                   local,
+                                                   stripe_getxattr_unwind,
+                                                   sub_volumes,
+                                                   local->marker.call_count,
+                                                   MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
+                                                   priv->vol_uuid)) {
+                                op_errno = EINVAL;
+                                goto err;
+                        }
+
+                        return 0;
+                }
+        }
+
+
+        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+
+        return 0;
+
+err:
+        STRIPE_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
 
-			if (data == FIRST_CHILD (this)) {
-				priv->first_child_down = 0;
-				default_notify (this, event, data);
-			}
-		}
-		UNLOCK (&priv->lock);
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		/* get an index number to set */
-		for (i = 0; i < priv->child_count; i++) {
-			if (data == priv->xl_array[i])
-				break;
-		}
-		priv->state[i] = 0;
-		for (i = 0; i < priv->child_count; i++) {
-			if (!priv->state[i])
-				down_client++;
-		}
+inline gf_boolean_t
+stripe_is_special_xattr (const char *name)
+{
+        gf_boolean_t    is_spl = _gf_false;
+
+        if (!name) {
+                goto out;
+        }
+
+        if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                      strlen (GF_XATTR_LOCKINFO_KEY))
+            || !strncmp (name, GF_XATTR_PATHINFO_KEY,
+                         strlen (GF_XATTR_PATHINFO_KEY)))
+                is_spl = _gf_true;
+out:
+        return is_spl;
+}
 
-		LOCK (&priv->lock);
-		{
-			priv->nodes_down = down_client;
+int32_t
+stripe_fgetxattr_from_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                const char *name, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv  = NULL;
+        int32_t           ret   = -1, op_errno = 0;
+        int               i     = 0;
+        xlator_list_t    *trav  = NULL;
+
+        priv = this->private;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->op_ret = -1;
+        frame->local = local;
+
+        strncpy (local->xsel, name, strlen (name));
+        local->nallocs = local->wind_count = priv->child_count;
+
+        for (i = 0, trav = this->children; i < priv->child_count; i++,
+                     trav = trav->next) {
+                STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
+                                   (void *) (long) i, trav->xlator,
+                                   trav->xlator->fops->fgetxattr,
+                                   fd, name, xdata);
+        }
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, NULL);
+        return ret;
+}
 
-			if (data == FIRST_CHILD (this)) {
-				priv->first_child_down = 1;
-				default_notify (this, event, data);
-			}
-		}
-		UNLOCK (&priv->lock);
-	}
-	break;
+int32_t
+stripe_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+        if (stripe_is_special_xattr (name)) {
+                stripe_fgetxattr_from_everyone (frame, this, fd, name, xdata);
+                goto out;
+        }
 
-	default:
-	{
-		/* */
-		default_notify (this, event, data);
-	}
-	break;
-	}
+        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
 
-	return 0;
+out:
+        return 0;
 }
-/**
- * init - This function is called when xlator-graph gets initialized. 
- *     The option given in volfiles are parsed here.
- * @this - 
- */
+
+
+
 int32_t
-init (xlator_t *this)
+stripe_priv_dump (xlator_t *this)
 {
-	stripe_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-	data_t *data = NULL;
-	int32_t count = 0;
+        char                    key[GF_DUMP_MAX_BUF_LEN];
+        int                     i = 0;
+        stripe_private_t       *priv = NULL;
+        int                     ret = -1;
+        struct stripe_options  *options = NULL;
 
-	trav = this->children;
-	while (trav) {
-		count++;
-		trav = trav->next;
-	}
+        GF_VALIDATE_OR_GOTO ("stripe", this, out);
 
-	if (!count) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"stripe configured without \"subvolumes\" option. "
-			"exiting");
-		return -1;
-	}
+        priv = this->private;
+        if (!priv)
+                goto out;
 
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	priv = CALLOC (1, sizeof (stripe_private_t));
-	ERR_ABORT (priv);
-	priv->xl_array = CALLOC (1, count * sizeof (xlator_t *));
-	ERR_ABORT (priv->xl_array);
-	priv->child_count = count;
-	LOCK_INIT (&priv->lock);
-
-	trav = this->children;
-	count = 0;
-	while (trav) {
-		priv->xl_array[count++] = trav->xlator;
-		trav = trav->next;
-	}
+        ret = TRY_LOCK (&priv->lock);
+        if (ret != 0)
+                goto out;
 
-	if (count > 256) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"maximum number of stripe subvolumes supported "
-			"is 256");
-		return -1;
-	}
+        gf_proc_dump_add_section("xlator.cluster.stripe.%s.priv", this->name);
+        gf_proc_dump_write("child_count","%d", priv->child_count);
 
-	priv->block_size = (128 * GF_UNIT_KB);
-	/* option stripe-pattern *avi:1GB,*pdf:4096 */
-	data = dict_get (this->options, "block-size");
-	if (!data) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"No \"option block-size <x>\" given, defaulting "
-			"to 128KB");
-	} else {
-		char *tmp_str = NULL;
-		char *tmp_str1 = NULL;
-		char *dup_str = NULL;
-		char *stripe_str = NULL;
-		char *pattern = NULL;
-		char *num = NULL;
-		struct stripe_options *temp_stripeopt = NULL;
-		struct stripe_options *stripe_opt = NULL;    
-
-		/* Get the pattern for striping. 
-		   "option block-size *avi:10MB" etc */
-		stripe_str = strtok_r (data->data, ",", &tmp_str);
-		while (stripe_str) {
-			dup_str = strdup (stripe_str);
-			stripe_opt = CALLOC (1, 
-					     sizeof (struct stripe_options));
-			ERR_ABORT (stripe_opt);
-			pattern = strtok_r (dup_str, ":", &tmp_str1);
-			num = strtok_r (NULL, ":", &tmp_str1);
-			if (num && 
-			    (gf_string2bytesize (num, 
-						 &stripe_opt->block_size) 
-			     != 0)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"invalid number format \"%s\"", 
-					num);
-				return -1;
-			} else if (!num && (gf_string2bytesize (
-						    pattern, 
-						    &stripe_opt->block_size) 
-					    != 0)) {
-				/* Possible that there is no pattern given */
-				stripe_opt->block_size = (128 * GF_UNIT_KB);
-				pattern = "*";
-			}
-			memcpy (stripe_opt->path_pattern, 
-				pattern, strlen (pattern));
-			
-			gf_log (this->name, GF_LOG_DEBUG, 
-				"block-size : pattern %s : size %"PRId64, 
-				stripe_opt->path_pattern, 
-				stripe_opt->block_size);
-			
-			if (!priv->pattern) {
-				priv->pattern = stripe_opt;
-			} else {
-				temp_stripeopt = priv->pattern;
-				while (temp_stripeopt->next)
-					temp_stripeopt = temp_stripeopt->next;
-				temp_stripeopt->next = stripe_opt;
-			}
-			stripe_str = strtok_r (NULL, ",", &tmp_str);
-		}
-	}
+        for (i = 0; i < priv->child_count; i++) {
+                sprintf (key, "subvolumes[%d]", i);
+                gf_proc_dump_write (key, "%s.%s", priv->xl_array[i]->type,
+                                    priv->xl_array[i]->name);
+        }
 
-	priv->xattr_supported = 1;
-	data = dict_get (this->options, "use-xattr");
-	if (data) {
-		if (gf_string2boolean (data->data, 
-				       &priv->xattr_supported) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"error setting hard check for extended "
-				"attribute");
-			//return -1;
-		}
-	}
+        options = priv->pattern;
+        while (options != NULL) {
+                gf_proc_dump_write ("path_pattern", "%s", priv->pattern->path_pattern);
+                gf_proc_dump_write ("options_block_size", "%ul", options->block_size);
 
-	/* notify related */
-	priv->nodes_down = priv->child_count;
-	this->private = priv;
+                options = options->next;
+        }
 
-	return 0;
-} 
+        gf_proc_dump_write ("block_size", "%ul", priv->block_size);
+        gf_proc_dump_write ("nodes-down", "%d", priv->nodes_down);
+        gf_proc_dump_write ("first-child_down", "%d", priv->first_child_down);
+        gf_proc_dump_write ("xattr_supported", "%d", priv->xattr_supported);
 
-/** 
- * fini -   Free all the private variables
- * @this - 
- */
-void 
-fini (xlator_t *this)
-{
-	stripe_private_t *priv = this->private;
-	struct stripe_options *prev = NULL;
-	struct stripe_options *trav = priv->pattern;
-	while (trav) {
-		prev = trav;
-		trav = trav->next;
-		FREE (prev);
-	}
-	FREE (priv->xl_array);
-	LOCK_DESTROY (&priv->lock);
-	FREE (priv);
-	return;
-}
+        UNLOCK (&priv->lock);
 
+out:
+        return ret;
+}
 
 struct xlator_fops fops = {
-	.stat        = stripe_stat,
-	.unlink      = stripe_unlink,
-	.symlink     = stripe_symlink,
-	.rename      = stripe_rename,
-	.link        = stripe_link,
-	.chmod       = stripe_chmod,
-	.chown       = stripe_chown,
-	.truncate    = stripe_truncate,
-	.utimens     = stripe_utimens,
-	.create      = stripe_create,
-	.open        = stripe_open,
-	.readv       = stripe_readv,
-	.writev      = stripe_writev,
-	.statfs      = stripe_statfs,
-	.flush       = stripe_flush,
-	.fsync       = stripe_fsync,
-	.setxattr    = stripe_setxattr,
-	.getxattr    = stripe_getxattr,
-	.removexattr = stripe_removexattr,
-	.access      = stripe_access,
-	.ftruncate   = stripe_ftruncate,
-	.fstat       = stripe_fstat,
-	.readlink    = stripe_readlink,
-	.mkdir       = stripe_mkdir,
-	.rmdir       = stripe_rmdir,
-	.lk          = stripe_lk,
-	.opendir     = stripe_opendir,
-	.fsyncdir    = stripe_fsyncdir,
-	.fchmod      = stripe_fchmod,
-	.fchown      = stripe_fchown,
-	.lookup      = stripe_lookup,
-	.setdents    = stripe_setdents,
-	.mknod       = stripe_mknod,
-};
-
-struct xlator_mops mops = {
-	.stats  = stripe_stats,
+        .stat           = stripe_stat,
+        .unlink         = stripe_unlink,
+        .rename         = stripe_rename,
+        .link           = stripe_link,
+        .truncate       = stripe_truncate,
+        .create         = stripe_create,
+        .open           = stripe_open,
+        .readv          = stripe_readv,
+        .writev         = stripe_writev,
+        .statfs         = stripe_statfs,
+        .flush          = stripe_flush,
+        .fsync          = stripe_fsync,
+        .ftruncate      = stripe_ftruncate,
+        .fstat          = stripe_fstat,
+        .mkdir          = stripe_mkdir,
+        .rmdir          = stripe_rmdir,
+        .lk             = stripe_lk,
+        .opendir        = stripe_opendir,
+        .fsyncdir       = stripe_fsyncdir,
+        .setattr        = stripe_setattr,
+        .fsetattr       = stripe_fsetattr,
+        .lookup         = stripe_lookup,
+        .mknod          = stripe_mknod,
+        .setxattr       = stripe_setxattr,
+        .fsetxattr      = stripe_fsetxattr,
+        .getxattr       = stripe_getxattr,
+        .fgetxattr      = stripe_fgetxattr,
+        .removexattr    = stripe_removexattr,
+        .fremovexattr   = stripe_fremovexattr,
+        .readdirp       = stripe_readdirp,
+	.fallocate	= stripe_fallocate,
+	.discard	= stripe_discard,
+        .zerofill       = stripe_zerofill,
 };
 
 struct xlator_cbks cbks = {
-	.release    = stripe_release,
-	.releasedir = stripe_releasedir
+        .release = stripe_release,
+        .forget  = stripe_forget,
 };
 
+struct xlator_dumpops dumpops = {
+        .priv = stripe_priv_dump,
+};
 
 struct volume_options options[] = {
-	{ .key  = {"block-size"}, 
-	  .type = GF_OPTION_TYPE_ANY 
-	},
-	{ .key  = {"use-xattr"}, 
-	  .type = GF_OPTION_TYPE_BOOL
+        { .key  = {"block-size"},
+          .type = GF_OPTION_TYPE_SIZE_LIST,
+          .default_value = "128KB",
+          .min = STRIPE_MIN_BLOCK_SIZE,
+          .description = "Size of the stripe unit that would be read "
+                         "from or written to the striped servers."
+        },
+        { .key  = {"use-xattr"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "true"
+        },
+	{ .key = {"coalesce"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "true",
+	  .description = "Enable/Disable coalesce mode to flatten striped "
+			 "files as stored on the server (i.e., eliminate holes "
+			 "caused by the traditional format)."
 	},
-	{ .key  = {NULL} },
+        { .key  = {NULL} },
 };
diff --git a/xlators/cluster/stripe/src/stripe.h b/xlators/cluster/stripe/src/stripe.h
new file mode 100644
index 000000000..5673d18f3
--- /dev/null
+++ b/xlators/cluster/stripe/src/stripe.h
@@ -0,0 +1,288 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _STRIPE_H_
+#define _STRIPE_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+#include "common-utils.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "stripe-mem-types.h"
+#include "libxlator.h"
+#include <fnmatch.h>
+#include <signal.h>
+
+#define STRIPE_PATHINFO_HEADER "STRIPE:"
+#define STRIPE_MIN_BLOCK_SIZE  (16*GF_UNIT_KB)
+
+#define STRIPE_STACK_UNWIND(fop, frame, params ...) do {           \
+                stripe_local_t *__local = NULL;                    \
+                if (frame) {                                       \
+                        __local = frame->local;                    \
+                        frame->local = NULL;                       \
+                }                                                  \
+                STACK_UNWIND_STRICT (fop, frame, params);          \
+                if (__local) {                                     \
+                        stripe_local_wipe(__local);                \
+                        mem_put (__local);       \
+                }                                                  \
+        } while (0)
+
+#define STRIPE_STACK_DESTROY(frame) do {                        \
+                stripe_local_t *__local = NULL;                 \
+                __local = frame->local;                         \
+                frame->local = NULL;                            \
+                STACK_DESTROY (frame->root);                    \
+                if (__local) {                                  \
+                        stripe_local_wipe (__local);            \
+                        mem_put (__local);    \
+                }                                               \
+        } while (0)
+
+#define STRIPE_VALIDATE_FCTX(fctx, label) do {                  \
+        int     idx = 0;                                        \
+        if (!fctx) {                                            \
+                op_errno = EINVAL;                              \
+                goto label;                                     \
+        }                                                       \
+        for (idx = 0; idx < fctx->stripe_count; idx++) {        \
+                if (!fctx->xl_array[idx]) {                     \
+                        gf_log (this->name, GF_LOG_ERROR,       \
+                                "fctx->xl_array[%d] is NULL",   \
+                                idx);                           \
+                        op_errno = ESTALE;                      \
+                        goto label;                             \
+                }                                               \
+        }                                                       \
+       } while (0)
+
+typedef struct stripe_xattr_sort {
+        int   pos;
+        int   xattr_len;
+        char *xattr_value;
+} stripe_xattr_sort_t;
+
+/**
+ * struct stripe_options : This keeps the pattern and the block-size
+ *     information, which is used for striping on a file.
+ */
+struct stripe_options {
+        struct stripe_options *next;
+        char                   path_pattern[256];
+        uint64_t               block_size;
+};
+
+/**
+ * Private structure for stripe translator
+ */
+struct stripe_private {
+        struct stripe_options  *pattern;
+        xlator_t              **xl_array;
+        uint64_t                block_size;
+        gf_lock_t               lock;
+        uint8_t                 nodes_down;
+        int8_t                  first_child_down;
+        int                    *last_event;
+        int8_t                  child_count;
+        gf_boolean_t            xattr_supported;  /* default yes */
+	gf_boolean_t		coalesce;
+        char                    vol_uuid[UUID_SIZE + 1];
+};
+
+/**
+ * Used to keep info about the replies received from readv/writev calls
+ */
+struct stripe_replies {
+        struct iovec *vector;
+        int32_t       count;    //count of vector
+        int32_t       op_ret;   //op_ret of readv
+        int32_t       op_errno;
+        int32_t       requested_size;
+        struct iatt   stbuf;    /* 'stbuf' is also a part of reply */
+};
+
+typedef struct _stripe_fd_ctx {
+        off_t      stripe_size;
+        int        stripe_count;
+	int	   stripe_coalesce;
+        int        static_array;
+        xlator_t **xl_array;
+} stripe_fd_ctx_t;
+
+
+/**
+ * Local structure to be passed with all the frames in case of STACK_WIND
+ */
+struct stripe_local; /* this itself is used inside the structure; */
+
+struct stripe_local {
+        struct stripe_local *next;
+        call_frame_t        *orig_frame;
+
+        stripe_fd_ctx_t     *fctx;
+
+        /* Used by _cbk functions */
+        struct iatt          stbuf;
+        struct iatt          pre_buf;
+        struct iatt          post_buf;
+        struct iatt          preparent;
+        struct iatt          postparent;
+
+        off_t                stbuf_size;
+        off_t                prebuf_size;
+        off_t                postbuf_size;
+        off_t                preparent_size;
+        off_t                postparent_size;
+
+        blkcnt_t             stbuf_blocks;
+        blkcnt_t             prebuf_blocks;
+        blkcnt_t             postbuf_blocks;
+        blkcnt_t             preparent_blocks;
+        blkcnt_t             postparent_blocks;
+
+        struct stripe_replies *replies;
+        struct statvfs        statvfs_buf;
+        dir_entry_t          *entry;
+
+        int8_t               revalidate;
+        int8_t               failed;
+        int8_t               unwind;
+
+        size_t               readv_size;
+        int32_t              entry_count;
+        int32_t              node_index;
+        int32_t              call_count;
+        int32_t              wind_count; /* used instead of child_cound
+                                            in case of read and write */
+        int32_t              op_ret;
+        int32_t              op_errno;
+        int32_t              count;
+        int32_t              flags;
+        char                *name;
+        inode_t             *inode;
+
+        loc_t                loc;
+        loc_t                loc2;
+
+        mode_t               mode;
+        dev_t                rdev;
+        /* For File I/O fops */
+        dict_t              *xdata;
+
+        stripe_xattr_sort_t *xattr_list;
+        int32_t              xattr_total_len;
+        int32_t              nallocs;
+        char xsel[256];
+
+        struct marker_str    marker;
+
+        /* General usage */
+        off_t                offset;
+        off_t                stripe_size;
+
+        int xattr_self_heal_needed;
+        int entry_self_heal_needed;
+
+        int8_t              *list;
+        struct gf_flock         lock;
+        fd_t                *fd;
+        void                *value;
+        struct iobref       *iobref;
+        gf_dirent_t          entries;
+        gf_dirent_t         *dirent;
+        dict_t              *xattr;
+        uuid_t               ia_gfid;
+
+        int                  xflag;
+        mode_t               umask;
+};
+
+typedef struct stripe_local   stripe_local_t;
+typedef struct stripe_private stripe_private_t;
+
+/*
+ * Determine the stripe index of a particular frame based on the translator.
+ */
+static inline int32_t stripe_get_frame_index(stripe_fd_ctx_t *fctx,
+					     call_frame_t *prev)
+{
+	int32_t i, idx = -1;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (fctx->xl_array[i] == prev->this) {
+			idx = i;
+			break;
+		}
+	}
+
+	return idx;
+}
+
+static inline void stripe_copy_xl_array(xlator_t **dst, xlator_t **src,
+					int count)
+{
+	int i;
+
+	for (i = 0; i < count; i++)
+		dst[i] = src[i];
+}
+
+void stripe_local_wipe (stripe_local_t *local);
+int32_t stripe_ctx_handle (xlator_t *this, call_frame_t *prev,
+                           stripe_local_t *local, dict_t *dict);
+void stripe_aggregate_xattr (dict_t *dst, dict_t *src);
+int32_t stripe_xattr_request_build (xlator_t *this, dict_t *dict,
+                                    uint64_t stripe_size, uint32_t stripe_count,
+                                    uint32_t stripe_index,
+				    uint32_t stripe_coalesce);
+int32_t stripe_get_matching_bs (const char *path, stripe_private_t *priv);
+int set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data);
+int32_t stripe_iatt_merge (struct iatt *from, struct iatt *to);
+int32_t stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
+                                    char **xattr_serz);
+int32_t stripe_free_xattr_str (stripe_local_t *local);
+int32_t stripe_xattr_aggregate (char *buffer, stripe_local_t *local,
+                                int32_t *total);
+off_t coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count);
+off_t uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
+			int stripe_index);
+int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz);
+
+/*
+ * Adjust the size attribute for files if coalesce is enabled.
+ */
+static inline void correct_file_size(struct iatt *buf, stripe_fd_ctx_t *fctx,
+	call_frame_t *prev)
+{
+	int index;
+
+	if (!IA_ISREG(buf->ia_type))
+		return;
+
+	if (!fctx || !fctx->stripe_coalesce)
+		return;
+
+	index = stripe_get_frame_index(fctx, prev);
+	buf->ia_size = uncoalesced_size(buf->ia_size, fctx->stripe_size,
+		fctx->stripe_count, index);
+}
+
+#endif /* _STRIPE_H_ */
diff --git a/xlators/cluster/unify/src/Makefile.am b/xlators/cluster/unify/src/Makefile.am
deleted file mode 100644
index b9e6f63e9..000000000
--- a/xlators/cluster/unify/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-
-xlator_LTLIBRARIES = unify.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
-
-unify_la_LDFLAGS = -module -avoidversion
-
-unify_la_SOURCES = unify.c unify-self-heal.c
-unify_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = unify.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/unify/src/unify-self-heal.c b/xlators/cluster/unify/src/unify-self-heal.c
deleted file mode 100644
index 4885dd91a..000000000
--- a/xlators/cluster/unify/src/unify-self-heal.c
+++ /dev/null
@@ -1,1225 +0,0 @@
-/*
-  Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * unify-self-heal.c : 
- *   This file implements few functions which enables 'unify' translator 
- *  to be consistent in its behaviour when 
- *     > a node fails, 
- *     > a node gets added, 
- *     > a failed node comes back
- *     > a new namespace server is added (ie, an fresh namespace server).
- * 
- *  This functionality of 'unify' will enable glusterfs to support storage
- *  system failure, and maintain consistancy. This works both ways, ie, when
- *  an entry (either file or directory) is found on namespace server, and not
- *  on storage nodes, its created in storage nodes and vica-versa.
- * 
- *  The two fops, where it can be implemented are 'getdents ()' and 'lookup ()'
- *
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "common-utils.h"
-
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count);
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count);
-
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count);
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count);
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		FREE (local->name);
-	}
-
-	if (local->sh_struct) {
-		if (local->sh_struct->offset_list)
-			FREE (local->sh_struct->offset_list);
-
-		if (local->sh_struct->entry_list)
-			FREE (local->sh_struct->entry_list);
-
-		if (local->sh_struct->count_list)
-			FREE (local->sh_struct->count_list);
-
-		FREE (local->sh_struct);
-	}
-
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-int32_t 
-unify_sh_setdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents on 
-		 * storagenodes is still pending.
-		 */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			if (!entry)
-				return 0;
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				FREE (trav->name);
-				if (S_ISLNK (trav->buf.st_mode))
-					FREE (trav->link);
-				FREE (trav);
-				trav = prev->next;
-			}
-			FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more entries
-				   to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_sh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			inode = local->loc1.inode;
-			fd_unref (local->fd);
-			tmp_dict = local->dict;
-
-			unify_local_wipe (local);
-			
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      inode, &local->stbuf, local->dict);
-			if (tmp_dict)
-				dict_unref (local->dict);
-		}
-	}
-  
-	return 0;
-}
-
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = CALLOC (1, sizeof (dir_entry_t));
-	
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, it 
-		   can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_sh_ns_setdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		if (local->sh_struct->entry_list[index]) {
-			prev = entry = local->sh_struct->entry_list[index];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				FREE (trav->name);
-				if (S_ISLNK (trav->buf.st_mode))
-					FREE (trav->link);
-				FREE (trav);
-				trav = prev->next;
-			}
-			FREE (entry);
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_sh_getdents_cbk -
- */
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = CALLOC (1, sizeof (dir_entry_t));
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_sh_opendir_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_WARNING, "failed");
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing
-			   storage nodes */
-			local->call_count--; 
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps
-				 * track of offset sent to each node during
-				 * STACK_WIND.
-				 */
-				local->sh_struct->offset_list = 
-					calloc (priv->child_count, 
-						sizeof (off_t));
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					calloc (priv->child_count, 
-						sizeof (dir_entry_t *));
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					calloc (priv->child_count, 
-						sizeof (int));
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_sh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-
-				/* did stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. */ 
-		inode = local->loc1.inode;
-		fd_unref (local->fd);
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-		/* Only 'self-heal' failed, lookup() was successful. */
-		local->op_ret = 0;
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, inode,
-			      &local->stbuf, local->dict);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * gf_sh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_sh_checksum_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       uint8_t *file_checksum,
-		       uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, ZR_FILENAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, ZR_FILENAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-                                        /* Using 'entry_count' as a flag */
-					local->entry_count = 1;
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, ZR_FILENAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < ZR_FILENAME_MAX; index++) {
-					/* Files should be present in
-					   only one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be
-					   same accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < ZR_FILENAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at directory level */
-			local->call_count = 0;
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		inode = local->loc1.inode;
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      inode,
-			      &local->stbuf,
-			      local->dict);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/* Foreground self-heal part over */
-
-/* Background self-heal part */
-
-int32_t 
-unify_bgsh_setdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents 
-		   on storagenodes is still pending. */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				FREE (trav->name);
-				if (S_ISLNK (trav->buf.st_mode))
-					FREE (trav->link);
-				FREE (trav);
-				trav = prev->next;
-			}
-			FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more
-				   entries to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_bgsh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			fd_unref (local->fd);
-			unify_local_wipe (local);
-			STACK_DESTROY (frame->root);
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = CALLOC (1, sizeof (dir_entry_t));
-
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, 
-		   it can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_bgsh_ns_setdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	if (local->sh_struct->entry_list[index]) {
-		prev = entry = local->sh_struct->entry_list[index];
-		if (!entry)
-			return 0;
-		trav = entry->next;
-		while (trav) {
-			prev->next = trav->next;
-			FREE (trav->name);
-			if (S_ISLNK (trav->buf.st_mode))
-				FREE (trav->link);
-			FREE (trav);
-			trav = prev->next;
-		}
-		FREE (entry);
-	}
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_bgsh_getdents_cbk -
- */
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = CALLOC (1, sizeof (dir_entry_t));
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for 
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_bgsh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_bgsh_opendir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			fd_t *fd)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int32_t callcnt = 0;
-	int16_t index = 0;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing 
-			   storage nodes */
-			local->call_count--; 
-			callcnt = local->call_count;
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps 
-				   track of offset sent to each node during 
-				   STACK_WIND. */
-				local->sh_struct->offset_list = 
-					calloc (priv->child_count, 
-						sizeof (off_t));
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					calloc (priv->child_count, 
-						sizeof (dir_entry_t *));
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					calloc (priv->child_count, 
-						sizeof (int));
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_bgsh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-				/* did a stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. 	 */
-		fd_unref (local->fd);
-		
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/**
- * gf_bgsh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_bgsh_checksum_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 uint8_t *file_checksum,
-			 uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, ZR_FILENAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, ZR_FILENAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other 
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-					/* Using 'entry_count' as a flag */
-					local->entry_count = 1; 
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, ZR_FILENAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < ZR_FILENAME_MAX; index++) {
-					/* Files should be present in only 
-					   one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be same 
-					   accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < ZR_FILENAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at the directory level */
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_bgsh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-      
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/* Background self-heal part over */
-
-
-
-
-/**
- * zr_unify_self_heal - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-zr_unify_self_heal (call_frame_t *frame,
-		    xlator_t *this,
-		    unify_local_t *local)
-{
-	unify_private_t *priv = this->private;
-	call_frame_t *bg_frame = NULL;
-	unify_local_t *bg_local = NULL;
-	inode_t *tmp_inode = NULL;
-	dict_t *tmp_dict = NULL;
-	int16_t index = 0;
-  
-	if (local->inode_generation < priv->inode_generation) {
-		/* Any self heal will be done at the directory level */
-		/* Update the inode's generation to the current generation
-		   value. */
-		local->inode_generation = priv->inode_generation;
-		inode_ctx_put (local->loc1.inode, this, 
-			  (uint64_t)(long)local->inode_generation);
-
-		if (priv->self_heal == ZR_UNIFY_FG_SELF_HEAL) {
-			local->op_ret = 0;
-			local->failed = 0;
-			local->call_count = priv->child_count + 1;
-			local->sh_struct = 
-				calloc (1, sizeof (struct unify_self_heal_struct));
-      
-			/* +1 is for NS */
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_checksum_cbk,
-						   priv->xl_array[index],
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->checksum,
-						   &local->loc1,
-						   0);
-			}
-
-			/* Self-heal in foreground, hence no need 
-			   to UNWIND here */
-			return 0;
-		}
-
-		/* Self Heal done in background */
-		bg_frame = copy_frame (frame);
-		INIT_LOCAL (bg_frame, bg_local);
-		loc_copy (&bg_local->loc1, &local->loc1);
-		bg_local->op_ret = 0;
-		bg_local->failed = 0;
-		bg_local->call_count = priv->child_count + 1;
-		bg_local->sh_struct = 
-			calloc (1, sizeof (struct unify_self_heal_struct));
-    
-		/* +1 is for NS */
-		for (index = 0; index < (priv->child_count + 1); index++) {
-			STACK_WIND_COOKIE (bg_frame,
-					   unify_bgsh_checksum_cbk,
-					   priv->xl_array[index],
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->checksum,
-					   &bg_local->loc1,
-					   0);
-		}
-	}
-
-	/* generation number matches, self heal already done or
-	 * self heal done in background: just do STACK_UNWIND 
-	 */
-	tmp_inode = local->loc1.inode;
-	tmp_dict = local->dict;
-
-	unify_local_wipe (local);
-
-	/* This is lookup_cbk ()'s UNWIND. */
-	STACK_UNWIND (frame,
-		      local->op_ret,
-		      local->op_errno,
-		      tmp_inode,
-		      &local->stbuf,
-		      local->dict);
-
-	if (tmp_dict)
-		dict_unref (tmp_dict);
-
-	return 0;
-}
-
diff --git a/xlators/cluster/unify/src/unify.c b/xlators/cluster/unify/src/unify.c
deleted file mode 100644
index e2a5e14b1..000000000
--- a/xlators/cluster/unify/src/unify.c
+++ /dev/null
@@ -1,4451 +0,0 @@
-/*
-  Copyright (c) 2006, 2007, 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * xlators/cluster/unify:
- *     - This xlator is one of the main translator in GlusterFS, which
- *   actually does the clustering work of the file system. One need to 
- *   understand that, unify assumes file to be existing in only one of 
- *   the child node, and directories to be present on all the nodes. 
- *
- * NOTE:
- *   Now, unify has support for global namespace, which is used to keep a 
- * global view of fs's namespace tree. The stat for directories are taken
- * just from the namespace, where as for files, just 'st_ino' is taken from
- * Namespace node, and other stat info is taken from the actual storage node.
- * Also Namespace node helps to keep consistant inode for files across 
- * glusterfs (re-)mounts.
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include <signal.h>
-#include <libgen.h>
-#include "compat-errno.h"
-#include "compat.h"
-
-#define UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR(_loc) do { \
-  if (!(_loc && _loc->inode)) {                            \
-    STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL);    \
-    return 0;                                              \
-  }                                                        \
-} while(0)
-
-
-#define UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!(_fd && !fd_ctx_get (_fd, this, NULL))) {       \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);      \
-    return 0;                                          \
-  }                                                    \
-} while(0)
-
-#define UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!_fd) {                                      \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);  \
-    return 0;                                      \
-  }                                                \
-} while(0)
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		FREE (local->name);
-	}
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-
-
-/*
- * unify_normalize_stats -
- */
-void
-unify_normalize_stats (struct statvfs *buf,
-		       unsigned long bsize,
-		       unsigned long frsize)
-{
-	double factor;
-
-	if (buf->f_bsize != bsize) {
-		factor = ((double) buf->f_bsize) / bsize;
-		buf->f_bsize  = bsize;
-		buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-		buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
-	}
-  
-	if (buf->f_frsize != frsize) {
-		factor = ((double) buf->f_frsize) / frsize;
-		buf->f_frsize = frsize;
-		buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-	}
-}
-
-
-xlator_t *
-unify_loc_subvol (loc_t *loc, xlator_t *this)
-{
-	unify_private_t *priv = NULL;
-	xlator_t        *subvol = NULL;
-	int16_t         *list = NULL;
-	long             index = 0;
-	xlator_t        *subvol_i = NULL;
-	int              ret = 0;
-	uint64_t         tmp_list = 0;
-
-	priv   = this->private;
-	subvol = NS (this);
-
-	if (!S_ISDIR (loc->inode->st_mode)) {
-		ret = inode_ctx_get (loc->inode, this, &tmp_list);
-		list = (int16_t *)(long)tmp_list;
-		if (!list)
-			goto out;
-
-		for (index = 0; list[index] != -1; index++) {
-			subvol_i = priv->xl_array[list[index]];
-			if (subvol_i != NS (this)) {
-				subvol = subvol_i;
-				break;
-			}
-		}
-	}
-out:
-	return subvol;
-}
-
-
-
-/**
- * unify_statfs_cbk -
- */
-int32_t
-unify_statfs_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct statvfs *stbuf)
-{
-	int32_t callcnt = 0;
-	struct statvfs *dict_buf = NULL;
-	unsigned long bsize;
-	unsigned long frsize;
-	unify_local_t *local = (unify_local_t *)frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			/* when a call is successfull, add it to local->dict */
-			dict_buf = &local->statvfs_buf;
-
-			if (dict_buf->f_bsize != 0) {
-				bsize  = max (dict_buf->f_bsize, 
-					      stbuf->f_bsize);
-
-				frsize = max (dict_buf->f_frsize, 
-					      stbuf->f_frsize);
-				unify_normalize_stats(dict_buf, bsize, frsize);
-				unify_normalize_stats(stbuf, bsize, frsize);
-			} else {
-				dict_buf->f_bsize   = stbuf->f_bsize;
-				dict_buf->f_frsize  = stbuf->f_frsize;
-			}
-      
-			dict_buf->f_blocks += stbuf->f_blocks;
-			dict_buf->f_bfree  += stbuf->f_bfree;
-			dict_buf->f_bavail += stbuf->f_bavail;
-			dict_buf->f_files  += stbuf->f_files;
-			dict_buf->f_ffree  += stbuf->f_ffree;
-			dict_buf->f_favail += stbuf->f_favail;
-			dict_buf->f_fsid    = stbuf->f_fsid;
-			dict_buf->f_flag    = stbuf->f_flag;
-			dict_buf->f_namemax = stbuf->f_namemax;
-			local->op_ret = op_ret;
-		} else {
-			/* fop on storage node has failed due to some error */
-			if (op_errno != ENOTCONN) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): %s", 
-					prev_frame->this->name, 
-					strerror (op_errno));
-			}
-			local->op_errno = op_errno;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->statvfs_buf);
-	}
-
-	return 0;
-}
-
-/**
- * unify_statfs -
- */
-int32_t
-unify_statfs (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-
-	INIT_LOCAL (frame, local);
-	local->call_count = ((unify_private_t *)this->private)->child_count;
-
-	while(trav) {
-		STACK_WIND (frame,
-			    unify_statfs_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->statfs,
-			    loc);
-		trav = trav->next;
-	}
-
-	return 0;
-}
-
-/**
- * unify_buf_cbk - 
- */
-int32_t
-unify_buf_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct stat *buf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s(): child(%s): path(%s): %s", 
-				gf_fop_list[frame->root->op],
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-
-			local->op_errno = op_errno;
-			if ((op_errno == ENOENT) && priv->optimist) 
-				local->op_ret = 0;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (NS (this) == prev_frame->this) {
-				local->st_ino = buf->st_ino;
-				/* If the entry is directory, get the stat
-				   from NS node */
-				if (S_ISDIR (buf->st_mode) || 
-				    !local->stbuf.st_blksize) {
-					local->stbuf = *buf;
-				}
-			}
-
-			if ((!S_ISDIR (buf->st_mode)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from Storage 
-				   node. */
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		/* If the inode number is not filled, operation should
-		   fail */
-		if (!local->st_ino)
-			local->op_ret = -1;
-
-		local->stbuf.st_ino = local->st_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-
-	return 0;
-}
-
-#define check_if_dht_linkfile(s) ((s->st_mode & ~S_IFMT) == S_ISVTX)
-
-/**
- * unify_lookup_cbk - 
- */
-int32_t 
-unify_lookup_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-		  struct stat *buf,
-		  dict_t *dict)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-	dict_t *local_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
- 
-		if (op_ret == -1) {
-			if ((op_errno != ENOTCONN) && (op_errno != ENOENT)) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-				local->op_errno = op_errno;
-				local->failed = 1;
-
-			} else if (local->revalidate && 
-				   !(priv->optimist && (op_errno == ENOENT))) {
-
-				gf_log (this->name, 
-					(op_errno == ENOTCONN) ? 
-					GF_LOG_DEBUG:GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-				local->op_errno = op_errno;
-				local->failed = 1;
-			}
-		}
-
-		if (op_ret == 0) {
-			local->op_ret = 0; 
-			
-			if (check_if_dht_linkfile(buf)) {
-				gf_log (this->name, GF_LOG_CRITICAL,
-					"file %s may be DHT link file on %s, "
-					"make sure the backend is not shared "
-					"between unify and DHT", 
-					local->loc1.path, 
-					priv->xl_array[(long)cookie]->name);
-			}
-
-			if (local->stbuf.st_mode && local->stbuf.st_blksize) {
-				/* make sure we already have a stbuf
-				   stored in local->stbuf */
-				if (S_ISDIR (local->stbuf.st_mode) && 
-				    !S_ISDIR (buf->st_mode)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on namespace, non-directory "
-						"on node '%s', returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-				if (!S_ISDIR (local->stbuf.st_mode) && 
-				    S_ISDIR (buf->st_mode)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on node '%s', non-directory "
-						"on namespace, returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-			}
-	
-			if (!local->revalidate && !S_ISDIR (buf->st_mode)) {
-				/* This is the first time lookup on file*/
-				if (!local->list) {
-					/* list is not allocated, allocate 
-					   the max possible range */
-					local->list = CALLOC (1, 2 * (priv->child_count + 2));
-					if (!local->list) {
-						gf_log (this->name, 
-							GF_LOG_CRITICAL, 
-							"Not enough memory");
-						STACK_UNWIND (frame, -1, 
-							      ENOMEM, inode, 
-							      NULL, NULL);
-						return 0;
-					}
-				}
-				/* update the index of the list */
-				local->list [local->index++] = 
-					(int16_t)(long)cookie;
-			}
-      
-			if ((!local->dict) && dict &&
-			    (priv->xl_array[(long)cookie] != NS(this)))	{
-				local->dict = dict_ref (dict);
-			}
-
-			/* index of NS node is == total child count */
-			if (priv->child_count == (int16_t)(long)cookie) {
-				/* Take the inode number from namespace */
-				local->st_ino = buf->st_ino;
-				if (S_ISDIR (buf->st_mode) || 
-				    !(local->stbuf.st_blksize)) {
-					local->stbuf = *buf;
-				}
-			} else if (!S_ISDIR (buf->st_mode)) {
-				/* If file, then get the stat from 
-				   storage node */
-				local->stbuf = *buf;
-			}
-
-			if (local->st_nlink < buf->st_nlink) {
-				local->st_nlink = buf->st_nlink;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local_dict = local->dict;
-		if (local->return_eio) {
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"[CRITICAL] Unable to fix the path (%s) with "
-				"self-heal, try manual verification. "
-				"returning EIO.", local->loc1.path);
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EIO, inode, NULL, NULL);
-			if (local_dict)	{
-				dict_unref (local_dict);
-			}
-			return 0;
-		}
-
-		if (!local->stbuf.st_blksize) {
-			/* Inode not present */
-			local->op_ret = -1;
-		} else {
-			if (!local->revalidate && 
-			    !S_ISDIR (local->stbuf.st_mode)) { 
-				/* If its a file, big array is useless, 
-				   allocate the smaller one */
-				int16_t *list = NULL;
-				list = CALLOC (1, 2 * (local->index + 1));
-				ERR_ABORT (list);
-				memcpy (list, local->list, 2 * local->index);
-				/* Make the end of the list as -1 */
-				FREE (local->list);
-				local->list = list;
-				local->list [local->index] = -1;
-				/* Update the inode's ctx with proper array */
-				/* TODO: log on failure */
-				inode_ctx_put (local->loc1.inode, this, 
-					       (uint64_t)(long)local->list);
-			}
-
-			if (S_ISDIR(local->loc1.inode->st_mode)) {
-				/* lookup is done for directory */
-				if (local->failed && priv->self_heal) {
-					/* Triggering self-heal */
-                                        /* means, self-heal required for this 
-					   inode */
-					local->inode_generation = 0; 
-					priv->inode_generation++;
-				}
-			} else {
-				local->stbuf.st_ino = local->st_ino;
-			}
-	  
-			local->stbuf.st_nlink = local->st_nlink;
-		}
-		if (local->op_ret == -1) {
-			if (!local->revalidate && local->list)
-				FREE (local->list);
-		}
-
-		if ((local->op_ret >= 0) && local->failed && 
-		    local->revalidate) {
-			/* Done revalidate, but it failed */
-			if (op_errno != ENOTCONN) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Revalidate failed for path(%s): %s", 
-					local->loc1.path, strerror (op_errno));
-			}
-			local->op_ret = -1;
-		}
-
-		if ((priv->self_heal && !priv->optimist) && 
-		    (!local->revalidate && (local->op_ret == 0) && 
-		     S_ISDIR(local->stbuf.st_mode))) {
-			/* Let the self heal be done here */
-			zr_unify_self_heal (frame, this, local);
-			local_dict = NULL;
-		} else {
-			/* either no self heal, or op_ret == -1 (failure) */
-			tmp_inode = local->loc1.inode;
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      tmp_inode, &local->stbuf, local->dict);
-		}
-		if (local_dict) {
-			dict_unref (local_dict);
-		}
-	}
-  
-	return 0;
-}
-
-/**
- * unify_lookup - 
- */
-int32_t 
-unify_lookup (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      dict_t *xattr_req)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	long index = 0;
-
-	if (!(loc && loc->inode)) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: Argument not right", loc?loc->path:"(null)");
-		STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL, NULL);
-		return 0;
-	}
-
-	if (!inode_ctx_get (loc->inode, this, NULL) && 
-	    loc->inode->st_mode && 
-	    !S_ISDIR (loc->inode->st_mode)) {
-		uint64_t tmp_list = 0;
-		/* check if revalidate or fresh lookup */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-	}
-
-	if (local->list) {
-		list = local->list;
-		for (index = 0; list[index] != -1; index++);
-		if (index != 2) {
-			if (index < 2) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning ESTALE for %s: file "
-					"count is %ld", loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-				unify_local_wipe (local);
-				STACK_UNWIND (frame, -1, ESTALE, 
-					      NULL, NULL, NULL);
-				return 0;  
-			} else {
-				/* There are more than 2 presences */
-				/* Just log and continue */
-				gf_log (this->name, GF_LOG_ERROR,
-					"%s: file count is %ld", 
-					loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-			}
-		}
-      
-		/* is revalidate */
-		local->revalidate = 1;
-      
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-      
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)(long)list[index], //cookie
-					   priv->xl_array [list[index]],
-					   priv->xl_array [list[index]]->fops->lookup,
-					   loc,
-					   xattr_req);
-			if (need_break)
-				break;
-		}
-	} else {
-		if (loc->inode->st_mode) {
-			if (inode_ctx_get (loc->inode, this, NULL)) {
-				inode_ctx_get (loc->inode, this, 
-					       &local->inode_generation);
-			}
-		}
-		/* This is first call, there is no list */
-		/* call count should be all child + 1 namespace */
-		local->call_count = priv->child_count + 1;
-      
-		for (index = 0; index <= priv->child_count; index++) {
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)index, //cookie
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   loc,
-					   xattr_req);
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_stat - if directory, get the stat directly from NameSpace child.
- *     if file, check for a hint and send it only there (also to NS).
- *     if its a fresh stat, then do it on all the nodes.
- *
- * NOTE: for all the call, sending cookie as xlator pointer, which will be 
- *       used in cbk.
- */
-int32_t
-unify_stat (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int16_t *list = NULL;
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL);
-		return 0;
-	}
-	local->st_ino = loc->inode->ino;
-	if (S_ISDIR (loc->inode->st_mode)) {
-		/* Directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->stat, loc);
-	} else {
-		/* File */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-    		list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-    
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->stat,
-				    loc);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_access_cbk -
- */
-int32_t
-unify_access_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_access - Send request to only namespace, which has all the 
- *      attributes set for the file.
- */
-int32_t
-unify_access (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t mask)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame,
-		    unify_access_cbk,
-		    NS(this),
-		    NS(this)->fops->access,
-		    loc,
-		    mask);
-
-	return 0;
-}
-
-int32_t
-unify_mkdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-		 struct stat *buf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-  
-		if ((op_ret == -1) && !(priv->optimist && 
-					(op_errno == ENOENT || 
-					 op_errno == EEXIST))) {
-			/* TODO: Decrement the inode_generation of 
-			 * this->inode's parent inode, hence the missing 
-			 * directory is created properly by self-heal. 
-			 * Currently, there is no way to get the parent 
-			 * inode directly.
-			 */
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			if (op_errno != EEXIST)
-				local->failed = 1;
-			local->op_errno = op_errno;
-		}
-  
-		if (op_ret >= 0)
-			local->op_ret = 0;
-
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (!local->failed) {
-			inode_ctx_put (local->loc1.inode, this, 
-				       priv->inode_generation);
-		}
-		
-		tmp_inode = local->loc1.inode;
-		unify_local_wipe (local);
-
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      tmp_inode, &local->stbuf);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_mkdir_cbk -
- */
-int32_t
-unify_ns_mkdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-		    struct stat *buf)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	long index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send mkdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR,
-			"namespace: path(%s): %s", 
-			local->name, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, NULL);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-
-	local->call_count = priv->child_count;
-
-	/* Send mkdir request to all the nodes now */
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND_COOKIE (frame,
-				   unify_mkdir_cbk,
-				   (void *)index, //cookie
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->mkdir,
-				   &local->loc1,
-				   local->mode);
-	}
-  
-	return 0;
-}
-
-
-/**
- * unify_mkdir -
- */
-int32_t
-unify_mkdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode)
-{
-	unify_local_t *local = NULL;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-
-	loc_copy (&local->loc1, loc);
-
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mkdir_cbk,
-		    NS(this),
-		    NS(this)->fops->mkdir,
-		    loc,
-		    mode);
-	return 0;
-}
-
-/**
- * unify_rmdir_cbk -
- */
-int32_t
-unify_rmdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0 || (priv->optimist && (op_errno == ENOENT)))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_rmdir_cbk -
- */
-int32_t
-unify_ns_rmdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-  
-	if (op_ret == -1) {
-		/* No need to send rmdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, 
-			((op_errno != ENOTEMPTY) ? 
-			 GF_LOG_ERROR : GF_LOG_DEBUG),
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno);
-		return 0;
-	}
-
-	local->call_count = priv->child_count;
-
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND (frame,
-			    unify_rmdir_cbk,
-			    priv->xl_array[index],
-			    priv->xl_array[index]->fops->rmdir,
-			    &local->loc1);
-	}
-
-	return 0;
-}
-
-/**
- * unify_rmdir -
- */
-int32_t
-unify_rmdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	unify_local_t *local = NULL;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_rmdir_cbk,
-		    NS(this),
-		    NS(this)->fops->rmdir,
-		    loc);
-
-	return 0;
-}
-
-/**
- * unify_open_cbk -
- */
-int32_t
-unify_open_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in 
-				   all the f*** / FileIO calls */
-				fd_ctx_set (fd, this, (uint64_t)(long)cookie);
-			}
-		}
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if ((local->failed == 1) && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			//local->op_errno = EIO; 
-      
-			if (!fd_ctx_get (local->fd, this, NULL)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on child node, "
-					"failed on namespace");
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on namespace, "
-					"failed on child node");
-			}
-		}
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local->fd);
-	}
-
-	return 0;
-}
-
-#ifdef GF_DARWIN_HOST_OS
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_open_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct stat *buf,
-		       dict_t *dict)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if ((op_ret == -1) && (op_errno != ENOENT)) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->index++;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->list[0] = (int16_t)(long)cookie;
-			} else {
-				local->list[1] = (int16_t)(long)cookie;
-			}
-			if (S_ISDIR (buf->st_mode))
-				local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		file_list[0] = local->list[0];
-		file_list[1] = local->list[1];
-		file_list[2] = -1;
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->name, local->index);
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning as file found on less "
-					"than 2 nodes");
-				STACK_UNWIND (frame, local->op_ret, 
-					      local->op_errno, local->fd);
-				return 0;
-			}
-		}
-
-		if (local->failed) {
-			/* Open on directory, return EISDIR */
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EISDIR, local->fd);
-			return 0;
-		}
-
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_open_readlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 const char *path)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		STACK_UNWIND (frame, -1, ENOENT);
-		return 0;
-	}
-
-	if (path[0] == '/') {
-		local->name = strdup (path);
-		ERR_ABORT (local->name);
-	} else {
-		char *tmp_str = strdup (local->loc1.path);
-		char *tmp_base = dirname (tmp_str);
-		local->name = CALLOC (1, ZR_PATH_MAX);
-		strcpy (local->name, tmp_base);
-		strncat (local->name, "/", 1);
-		strcat (local->name, path);
-		FREE (tmp_str);
-	}
-  
-	local->list = CALLOC (1, sizeof (int16_t) * 3);
-	ERR_ABORT (local->list);
-	local->call_count = priv->child_count + 1;
-	local->op_ret = -1;
-	for (index = 0; index <= priv->child_count; index++) {
-		/* Send the lookup to all the nodes including namespace */
-		STACK_WIND_COOKIE (frame,
-				   unify_open_lookup_cbk,
-				   (void *)(long)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->lookup,
-				   &local->loc1,
-				   NULL);
-	}
-
-	return 0;
-}
-#endif /* GF_DARWIN_HOST_OS */
-
-/**
- * unify_open - 
- */
-int32_t
-unify_open (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t flags,
-	    fd_t *fd)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t file_list[3] = {0,};
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Init */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->fd    = fd;
-	local->flags = flags;
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	local->list = list;
-	file_list[0] = priv->child_count; /* Thats namespace */
-	file_list[2] = -1;
-	for (index = 0; list[index] != -1; index++) {
-		local->call_count++;
-		if (list[index] != priv->child_count)
-			file_list[1] = list[index];
-	}
-
-	if (local->call_count != 2) {
-		/* If the lookup was done for file */
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: entry_count is %d",
-			loc->path, local->call_count);
-		for (index = 0; local->list[index] != -1; index++)
-			gf_log (this->name, GF_LOG_ERROR, "%s: found on %s",
-				loc->path, priv->xl_array[list[index]]->name);
-
-		if (local->call_count < 2) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"returning EIO as file found on onlyone node");
-			STACK_UNWIND (frame, -1, EIO, fd);
-			return 0;
-		}
-	}
-
-#ifdef GF_DARWIN_HOST_OS
-	/* Handle symlink here */
-	if (S_ISLNK (loc->inode->st_mode)) {
-		/* Callcount doesn't matter here */
-		STACK_WIND (frame,
-			    unify_open_readlink_cbk,
-			    NS(this),
-			    NS(this)->fops->readlink,
-			    loc, ZR_PATH_MAX);
-		return 0;
-	}
-#endif /* GF_DARWIN_HOST_OS */
-
-	local->call_count = 2;
-	for (index = 0; file_list[index] != -1; index++) {
-		char need_break = (file_list[index+1] == -1);
-		STACK_WIND_COOKIE (frame,
-				   unify_open_cbk,
-				   priv->xl_array[file_list[index]], //cookie
-				   priv->xl_array[file_list[index]],
-				   priv->xl_array[file_list[index]]->fops->open,
-				   loc,
-				   flags,
-				   fd);
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-
-int32_t 
-unify_create_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	unify_local_t *local = frame->local;
-	inode_t *inode = local->loc1.inode;
-
-	unify_local_wipe (local);
-
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      inode, &local->stbuf);
-  
-	return 0;
-}
-
-/**
- * unify_create_open_cbk -
- */
-int32_t
-unify_create_open_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       fd_t *fd)
-{
-	int ret = 0;
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_value = 0;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in all 
-				   the f*** / FileIO calls */
-				/* TODO: log on failure */
-				ret = fd_ctx_get (fd, this, &tmp_value);
-				cookie = (void *)(long)tmp_value;
-			} else {
-				/* NOTE: open successful on namespace.
-				 *       fd's ctx can be used to identify open 
-				 *       failure on storage subvolume. cool 
-				 *       ide ;) */
-				local->failed = 0;
-			}
-		} else {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				((xlator_t *)cookie)->name,
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed == 1 && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			local->fd = fd;
-			local->call_count = 1;
-
-			if (!fd_ctx_get (local->fd, this, &tmp_value)) {
-				child = (xlator_t *)(long)tmp_value;
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on child node, "
-					"failed on namespace");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    child,
-					    child->fops->unlink,
-					    &local->loc1);
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on namespace, "
-					"failed on child node");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    NS(this),
-					    NS(this)->fops->unlink,
-					    &local->loc1);
-			}
-			return 0;
-		}
-		inode = local->loc1.inode;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, fd,
-			      inode, &local->stbuf);
-	}
-	return 0;
-}
-
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_create_lookup_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 inode_t *inode,
-			 struct stat *buf,
-			 dict_t *dict)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->list[local->index++] = (int16_t)(long)cookie;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->st_ino = buf->st_ino;
-			} else {
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t *list = local->list;
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		local->list [local->index] = -1;
-		file_list[0] = list[0];
-		file_list[1] = list[1];
-		file_list[2] = -1;
-
-		local->stbuf.st_ino = local->st_ino;
-		/* TODO: log on failure */
-		inode_ctx_put (local->loc1.inode, this, 
-			       (uint64_t)(long)local->list);
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->loc1.path, local->index);
-			file_list[0] = priv->child_count;
-			for (index = 0; list[index] != -1; index++) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"%s: found on %s", local->loc1.path, 
-					priv->xl_array[list[index]]->name);
-				if (list[index] != priv->child_count)
-					file_list[1] = list[index];
-			}
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning EIO as file found on "
-					"only one node");
-				STACK_UNWIND (frame, -1, EIO, 
-					      local->fd, inode, NULL);
-				return 0;
-			}
-		}
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_create_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_create_cbk -
- */
-int32_t
-unify_create_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd,
-		  inode_t *inode,
-		  struct stat *buf)
-{
-	int ret = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	inode_t *tmp_inode = NULL;
-
-	if (op_ret == -1) {
-		/* send unlink () on Namespace */
-		local->op_errno = op_errno;
-		local->op_ret = -1;
-		local->call_count = 1;
-		gf_log (this->name, GF_LOG_ERROR,
-			"create failed on %s (file %s, error %s), "
-			"sending unlink to namespace", 
-			prev_frame->this->name, 
-			local->loc1.path, strerror (op_errno));
-
-		STACK_WIND (frame,
-			    unify_create_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-
-		return 0;
-	}
-
-	if (op_ret >= 0) {
-		local->op_ret = op_ret;
-		local->stbuf = *buf;
-		/* Just inode number should be from NS node */
-		local->stbuf.st_ino = local->st_ino;
-
-		/* TODO: log on failure */
-		ret = fd_ctx_set (fd, this, (uint64_t)(long)prev_frame->this);
-	}
-  
-	tmp_inode = local->loc1.inode;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      tmp_inode, &local->stbuf);
-
-	return 0;
-}
-
-/**
- * unify_ns_create_cbk -
- * 
- */
-int32_t
-unify_ns_create_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd,
-		     inode_t *inode,
-		     struct stat *buf)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send create request to other servers, as 
-		   namespace action failed. Handle exclusive create here. */
-		if ((op_errno != EEXIST) || 
-		    ((op_errno == EEXIST) && 
-		     ((local->flags & O_EXCL) == O_EXCL))) {
-			/* If its just a create call without O_EXCL, 
-			   don't do this */
-			gf_log (this->name, GF_LOG_ERROR,
-				"namespace: path(%s): %s", 
-				local->loc1.path, strerror (op_errno));
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
-			return 0;
-		}
-	}
-  
-	if (op_ret >= 0) {
-		/* Get the inode number from the NS node */
-		local->st_ino = buf->st_ino;
-  
-		local->op_ret = -1;
-
-		/* Start the mapping list */
-		list = CALLOC (1, sizeof (int16_t) * 3);
-		ERR_ABORT (list);
-		inode_ctx_put (inode, this, (uint64_t)(long)list);
-		list[0] = priv->child_count;
-		list[2] = -1;
-
-		/* This means, file doesn't exist anywhere in the Filesystem */
-		sched_ops = priv->sched_ops;
-
-		/* Send create request to the scheduled node now */
-		sched_xl = sched_ops->schedule (this, local->loc1.path);
-		if (sched_xl == NULL)
-		{
-			/* send unlink () on Namespace */
-			local->op_errno = ENOTCONN;
-			local->op_ret = -1;
-			local->call_count = 1;
-			gf_log (this->name, GF_LOG_ERROR,
-				"no node online to schedule create:(file %s) "
-				"sending unlink to namespace", 
-				(local->loc1.path)?local->loc1.path:"");
-
-			STACK_WIND (frame,
-				    unify_create_unlink_cbk,
-				    NS(this),
-				    NS(this)->fops->unlink,
-				    &local->loc1);
-	
-			return 0;
-		}
-
-		for (index = 0; index < priv->child_count; index++)
-			if (sched_xl == priv->xl_array[index])
-				break;
-		list[1] = index;
-
-		STACK_WIND (frame, unify_create_cbk,
-			    sched_xl, sched_xl->fops->create,
-			    &local->loc1, local->flags, local->mode, fd);
-	} else {
-		/* File already exists, and there is no O_EXCL flag */
-
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"File(%s) already exists on namespace, sending "
-			"open instead", local->loc1.path);
-
-		local->list = CALLOC (1, sizeof (int16_t) * 3);
-		ERR_ABORT (local->list);
-		local->call_count = priv->child_count + 1;
-		local->op_ret = -1;
-		for (index = 0; index <= priv->child_count; index++) {
-			/* Send lookup() to all nodes including namespace */
-			STACK_WIND_COOKIE (frame,
-					   unify_create_lookup_cbk,
-					   (void *)(long)index,
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   &local->loc1,
-					   NULL);
-		}
-	}
-	return 0;
-}
-
-/**
- * unify_create - create a file in global namespace first, so other 
- *    clients can see them. Create the file in storage nodes in background.
- */
-int32_t
-unify_create (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flags,
-	      mode_t mode,
-	      fd_t *fd)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->flags = flags;
-	local->fd = fd;
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, fd, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_create_cbk,
-		    NS(this),
-		    NS(this)->fops->create,
-		    loc,
-		    flags | O_EXCL,
-		    mode,
-		    fd);
-  
-	return 0;
-}
-
-
-/**
- * unify_opendir_cbk - 
- */
-int32_t
-unify_opendir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-
-	return 0;
-}
-
-/** 
- * unify_opendir -
- */
-int32_t
-unify_opendir (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       fd_t *fd)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame, unify_opendir_cbk,
-		    NS(this), NS(this)->fops->opendir, loc, fd);
-
-	return 0;
-}
-
-
-/**
- * unify_chmod - 
- */
-int32_t
-unify_chmod (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	uint64_t tmp_list = 0;
-		
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, loc);
-	local->st_ino = loc->inode->ino;
-
-	if (S_ISDIR (loc->inode->st_mode)) {
-		local->call_count = priv->child_count + 1;
-      
-		for (index = 0; index < (priv->child_count + 1); index++) {
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->chmod,
-				    loc, mode);
-		}    
-	} else {
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		for (index = 0; local->list[index] != -1; index++) {
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[local->list[index]],
-				    priv->xl_array[local->list[index]]->fops->chmod,
-				    loc,
-				    mode);
-			if (!--callcnt)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_chown - 
- */
-int32_t
-unify_chown (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     uid_t uid,
-	     gid_t gid)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->st_ino = loc->inode->ino;
-
-	if (S_ISDIR (loc->inode->st_mode)) {
-		local->call_count = priv->child_count + 1;
-      
-		for (index = 0; index < (priv->child_count + 1); index++) {
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->chown,
-				    loc, uid, gid);
-		}    
-	} else {
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		for (index = 0; local->list[index] != -1; index++) {
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[local->list[index]],
-				    priv->xl_array[local->list[index]]->fops->chown,
-				    loc, uid, gid);
-			if (!--callcnt)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_truncate_cbk - 
- */
-int32_t
-unify_truncate_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct stat *buf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-			if (!((op_errno == ENOENT) && priv->optimist))
-				local->op_ret = -1;
-		}
-
-		if (op_ret >= 0) {
-			if (NS (this) == prev_frame->this) {
-				local->st_ino = buf->st_ino;
-				/* If the entry is directory, get the 
-				   stat from NS node */
-				if (S_ISDIR (buf->st_mode) || 
-				    !local->stbuf.st_blksize) {
-					local->stbuf = *buf;
-				}
-			}
-
-			if ((!S_ISDIR (buf->st_mode)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from 
-				   Storage node. */
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		if (local->st_ino)
-			local->stbuf.st_ino = local->st_ino;
-		else
-			local->op_ret = -1;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-
-	return 0;
-}
-
-/**
- * unify_truncate - 
- */
-int32_t
-unify_truncate (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		off_t offset)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->st_ino = loc->inode->ino;
-
-	if (S_ISDIR (loc->inode->st_mode)) {
-		local->call_count = 1;
-      
-		STACK_WIND (frame,
-			    unify_buf_cbk,
-			    NS(this),
-			    NS(this)->fops->stat,
-			    loc);
-	} else {
-		local->op_ret = 0;
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		/* Don't send truncate to NS node */
-		STACK_WIND (frame, unify_truncate_cbk, NS(this),
-			    NS(this)->fops->stat, loc);
-		callcnt--;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[local->list[index]]) {
-				STACK_WIND (frame,
-					    unify_truncate_cbk,
-					    priv->xl_array[local->list[index]],
-					    priv->xl_array[local->list[index]]->fops->truncate,
-					    loc,
-					    offset);
-				if (!--callcnt)
-					break;
-			}
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_utimens - 
- */
-int32_t 
-unify_utimens (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       struct timespec tv[2])
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-  
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->st_ino = loc->inode->ino;
-
-	if (S_ISDIR (loc->inode->st_mode)) {
-		local->call_count = priv->child_count + 1;
-      
-		for (index = 0; index < (priv->child_count + 1); index++) {
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->utimens,
-				    loc, tv);
-		}
-	} else {
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		for (index = 0; local->list[index] != -1; index++) {
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[local->list[index]],
-				    priv->xl_array[local->list[index]]->fops->utimens,
-				    loc,
-				    tv);
-			if (!--callcnt)
-				break;
-		}
-	}
-  
-	return 0;
-}
-
-/**
- * unify_readlink_cbk - 
- */
-int32_t
-unify_readlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    const char *path)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, path);
-	return 0;
-}
-
-/**
- * unify_readlink - Read the link only from the storage node.
- */
-int32_t
-unify_readlink (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		size_t size)
-{
-	unify_private_t *priv = this->private;
-	int32_t entry_count = 0;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-  
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		entry_count++;
-
-	if (entry_count >= 2) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_readlink_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->readlink,
-					    loc,
-					    size);
-				break;
-			}
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"returning ENOENT, no softlink files found "
-			"on storage node");
-		STACK_UNWIND (frame, -1, ENOENT, NULL);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink_cbk - 
- */
-int32_t
-unify_unlink_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0  || ((op_errno == ENOENT) && priv->optimist))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink - 
- */
-int32_t
-unify_unlink (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		local->call_count++;
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_unlink_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->unlink,
-				    loc);
-			if (need_break)
-				break;
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: returning ENOENT", loc->path);
-		STACK_UNWIND (frame, -1, ENOENT);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_readv_cbk - 
- */
-int32_t
-unify_readv_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iovec *vector,
-		 int32_t count,
-		 struct stat *stbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf);
-	return 0;
-}
-
-/**
- * unify_readv - 
- */
-int32_t
-unify_readv (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     size_t size,
-	     off_t offset)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_readv_cbk,
-		    child,
-		    child->fops->readv,
-		    fd,
-		    size,
-		    offset);
-
-
-	return 0;
-}
-
-/**
- * unify_writev_cbk - 
- */
-int32_t
-unify_writev_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct stat *stbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, stbuf);
-	return 0;
-}
-
-/**
- * unify_writev - 
- */
-int32_t
-unify_writev (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      struct iovec *vector,
-	      int32_t count,
-	      off_t off)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_writev_cbk,
-		    child,
-		    child->fops->writev,
-		    fd,
-		    vector,
-		    count,
-		    off);
-
-	return 0;
-}
-
-/**
- * unify_ftruncate -
- */
-int32_t
-unify_ftruncate (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 off_t offset)
-{
-	xlator_t *child = NULL;
-	unify_local_t *local = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->op_ret = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	local->call_count = 2;
-  
-	STACK_WIND (frame, unify_truncate_cbk, 
-		    child, child->fops->ftruncate,
-		    fd, offset);
-  
-	STACK_WIND (frame, unify_truncate_cbk, 
-		    NS(this), NS(this)->fops->fstat,
-		    fd);
-  
-	return 0;
-}
-
-
-/**
- * unify_fchmod - 
- */
-int32_t 
-unify_fchmod (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      mode_t mode)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;	
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->st_ino = fd->inode->ino;
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_buf_cbk, child, 
-			    child->fops->fchmod, fd, mode);
-
-		STACK_WIND (frame, unify_buf_cbk, NS(this),	
-			    NS(this)->fops->fchmod, fd, mode);
-
-	} else {
-		/* this is an directory */
-		local->call_count = 1;
-    
-		STACK_WIND (frame, unify_buf_cbk,
-			    NS(this), NS(this)->fops->fchmod, fd, mode);
-	}
-
-	return 0;
-}
-
-/**
- * unify_fchown - 
- */
-int32_t 
-unify_fchown (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      uid_t uid,
-	      gid_t gid)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->st_ino = fd->inode->ino;
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_buf_cbk, child,
-			    child->fops->fchown, fd, uid, gid);
-
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fchown,	fd, uid, gid);
-	} else {
-		local->call_count = 1;
-    
-		STACK_WIND (frame, unify_buf_cbk,
-			    NS(this), NS(this)->fops->fchown,
-			    fd, uid, gid);
-	}
-  
-	return 0;
-}
-
-/**
- * unify_flush_cbk - 
- */
-int32_t
-unify_flush_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_flush -
- */
-int32_t
-unify_flush (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_flush_cbk, child, 
-		    child->fops->flush, fd);
-
-	return 0;
-}
-
-
-/**
- * unify_fsync_cbk - 
- */
-int32_t
-unify_fsync_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_fsync - 
- */
-int32_t
-unify_fsync (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fsync_cbk, child,
-		    child->fops->fsync, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_fstat - Send fstat FOP to Namespace only if its directory, and to 
- *     both namespace and the storage node if its a file.
- */
-int32_t
-unify_fstat (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	INIT_LOCAL (frame, local);
-	local->st_ino = fd->inode->ino;
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_buf_cbk, child,
-			    child->fops->fstat, fd);
-
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-
-	} else {
-		/* this is an directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-	}
-
-	return 0;
-}
-
-/**
- * unify_getdents_cbk - 
- */
-int32_t
-unify_getdents_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dir_entry_t *entry,
-		    int32_t count)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entry, count);
-	return 0;
-}
-
-/**
- * unify_getdents - send the FOP request to all the nodes.
- */
-int32_t
-unify_getdents (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		size_t size,
-		off_t offset,
-		int32_t flag)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_getdents_cbk, NS(this),
-		    NS(this)->fops->getdents, fd, size, offset, flag);
-
-	return 0;
-}
-
-
-/**
- * unify_readdir_cbk - 
- */
-int32_t
-unify_readdir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-/**
- * unify_readdir - send the FOP request to all the nodes.
- */
-int32_t
-unify_readdir (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       size_t size,
-	       off_t offset)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_readdir_cbk, NS(this),
-		    NS(this)->fops->readdir, fd, size, offset);
-
-	return 0;
-}
-
-
-/**
- * unify_fsyncdir_cbk - 
- */
-int32_t
-unify_fsyncdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-
-	return 0;
-}
-
-/**
- * unify_fsyncdir -
- */
-int32_t
-unify_fsyncdir (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		int32_t flags)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_fsyncdir_cbk,
-		    NS(this), NS(this)->fops->fsyncdir, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_lk_cbk - UNWIND frame with the proper return arguments.
- */
-int32_t
-unify_lk_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct flock *lock)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, lock);
-	return 0;
-}
-
-/**
- * unify_lk - Send it to all the storage nodes, (should be 1) which has file.
- */
-int32_t
-unify_lk (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  int32_t cmd,
-	  struct flock *lock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_lk_cbk, child,
-		    child->fops->lk, fd, cmd, lock);
-
-	return 0;
-}
-
-
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno);
-
-static int32_t
-unify_setxattr_file_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	unify_private_t *private = this->private;
-	unify_local_t *local = frame->local;
-	xlator_t *sched_xl = NULL;
-	struct sched_ops *sched_ops = NULL;
-
-	if (op_ret == -1) {
-		if (!ENOTSUP)
-			gf_log (this->name, GF_LOG_ERROR,
-				"setxattr with XATTR_CREATE on ns: "
-				"path(%s) key(%s): %s",
-				local->loc1.path, local->name, 
-				strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno);
-		return 0;
-	} 
-
-	LOCK (&frame->lock);
-	{
-		local->failed = 0;
-		local->op_ret = 0;
-		local->op_errno = 0;
-		local->call_count = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	/* schedule XATTR_CREATE on one of the child node */
-	sched_ops = private->sched_ops;
-    
-	/* Send create request to the scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->name); 
-	if (!sched_xl) {
-		STACK_UNWIND (frame, -1, ENOTCONN);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_setxattr_cbk,
-		    sched_xl,
-		    sched_xl->fops->setxattr,
-		    &local->loc1,
-		    local->dict,
-		    local->flags);
-	return 0;
-}
-
-/**
- * unify_setxattr_cbk - When all the child nodes return, UNWIND frame.
- */
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	dict_t *dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, (((op_errno == ENOENT) || 
-					      (op_errno == ENOTSUP))? 
-					     GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			if (local->failed == -1) {
-				local->failed = 1;
-			}
-			local->op_errno = op_errno;
-		} else {
-			local->failed = 0;
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed && local->name && 
-		    ZR_FILE_CONTENT_REQUEST(local->name)) {      
-			dict = get_new_dict ();
-			dict_set (dict, local->dict->members_list->key, 
-				  data_from_dynptr(NULL, 0));
-			dict_ref (dict);
-
-			local->call_count = 1;
-
-			STACK_WIND (frame,
-				    unify_setxattr_file_cbk,
-				    NS(this),
-				    NS(this)->fops->setxattr,
-				    &local->loc1,
-				    dict,
-				    XATTR_CREATE);
-
-			dict_unref (dict);
-			return 0;
-		}
-    
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sexattr - This function should be sent to all the storage nodes, 
- *       which contains the file, (excluding namespace).
- */
-int32_t
-unify_setxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		dict_t *dict,
-		int32_t flags)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-	data_pair_t *trav = dict->members_list;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->failed = -1;
-	loc_copy (&local->loc1, loc);
-
-	if (S_ISDIR (loc->inode->st_mode)) {
-
-		if (trav && trav->key && ZR_FILE_CONTENT_REQUEST(trav->key)) {
-			/* direct the storage xlators to change file 
-			   content only if file exists */
-			local->flags = flags;
-			local->dict = dict;
-			local->name = strdup (trav->key);
-			flags |= XATTR_REPLACE;
-		}
-
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_setxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->setxattr,
-				    loc, dict, flags);
-		}
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-  
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_setxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->setxattr,
-					    loc,
-					    dict,
-					    flags);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	}
-
-	/* No entry in storage nodes */
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"returning ENOENT, file not found on storage node.");
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-/**
- * unify_getxattr_cbk - This function is called from only one child, so, no
- *     need of any lock or anything else, just send it to above layer 
- */
-int32_t
-unify_getxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dict_t *value)
-{
-	int32_t callcnt = 0;
-	dict_t *local_value = NULL;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, 
-				(((op_errno == ENOENT) || 
-				  (op_errno == ENODATA) || 
-				  (op_errno == ENOTSUP)) ? 
-				 GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-		} else {
-			if (!local->dict)
-				local->dict = dict_ref (value);
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local_value = local->dict;
-		local->dict = NULL;
-      
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local_value);
-      
-		if (local_value)
-			dict_unref (local_value);
-	} 
-
-	return 0;
-}
-
-
-/** 
- * unify_getxattr - This FOP is sent to only the storage node.
- */
-int32_t
-unify_getxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		const char *name)
-{
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t count = 0;
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	INIT_LOCAL (frame, local);
-
-	if (S_ISDIR (loc->inode->st_mode)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame,
-				    unify_getxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->getxattr,
-				    loc,
-				    name);
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			count++;
-		}
-	}
-
-	if (count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_getxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->getxattr,
-					    loc,
-					    name);
-				if (!--count)
-					break;
-			}
-		}
-	} else {
-		dict_t *tmp_dict = get_new_dict ();
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"%s: returning ENODATA, no file found on storage node",
-			loc->path);
-		STACK_UNWIND (frame, -1, ENODATA, tmp_dict);
-		dict_destroy (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr_cbk - Wait till all the child node returns the call
- *      and then UNWIND to above layer.
- */
-int32_t
-unify_removexattr_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{ 
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			if (op_errno != ENOTSUP)
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): path(%s): %s", 
-					prev_frame->this->name, 
-					local->loc1.path, strerror (op_errno));
-		} else {
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);  
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr - Send it to all the child nodes which has the files.
- */
-int32_t
-unify_removexattr (call_frame_t *frame,
-		   xlator_t *this,
-		   loc_t *loc,
-		   const char *name)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	if (S_ISDIR (loc->inode->st_mode)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame, 		    
-				    unify_removexattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->removexattr,
-				    loc,
-				    name);
-
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_removexattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->removexattr,
-					    loc,
-					    name);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	} 
-
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"%s: returning ENOENT, not found on storage node.", loc->path);
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-int32_t 
-unify_mknod_unlink_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-  
-	unify_local_wipe (local);
-	/* No log required here as this -1 is for mknod call */
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_mknod_cbk - 
- */
-int32_t
-unify_mknod_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-		 struct stat *buf)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, sending unlink to "
-			"namespace");
-		local->op_errno = op_errno;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.st_ino = local->st_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf);
-	return 0;
-}
-
-/**
- * unify_ns_mknod_cbk - 
- */
-int32_t
-unify_ns_mknod_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-		    struct stat *buf)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	call_frame_t *prev_frame = cookie;
-
-	if (op_ret == -1) {
-		/* No need to send mknod request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s): %s", 
-			prev_frame->this->name, local->loc1.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-	local->st_ino = buf->st_ino;
-
-	list = CALLOC (1, sizeof (int16_t) * 3);
-	ERR_ABORT (list);
-	list[0] = priv->child_count;
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send mknod request to scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, no node online "
-			"at the moment, sending unlink to NS");
-		local->op_errno = ENOTCONN;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-  
-	STACK_WIND (frame,  unify_mknod_cbk,
-		    sched_xl,  sched_xl->fops->mknod,
-		    &local->loc1, local->mode, local->dev);
-
-	return 0;
-}
-
-/**
- * unify_mknod - Create a device on namespace first, and later create on 
- *       the storage node.
- */
-int32_t
-unify_mknod (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode,
-	     dev_t rdev)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->dev = rdev;
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mknod_cbk,
-		    NS(this),
-		    NS(this)->fops->mknod,
-		    loc,
-		    mode,
-		    rdev);
-
-	return 0;
-}
-
-int32_t 
-unify_symlink_unlink_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno)
-{
-	unify_local_t *local = frame->local;
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_symlink_cbk - 
- */
-int32_t
-unify_symlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-		   struct stat *buf)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, sending unlink "
-			"to namespace");
-
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-    
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.st_ino = local->st_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf);
-
-	return 0;
-}
-
-/**
- * unify_ns_symlink_cbk - 
- */
-int32_t
-unify_ns_symlink_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-		      struct stat *buf)
-{
-
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	int16_t *list = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send symlink request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, NULL, buf);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->st_ino = buf->st_ino;
-  
-	/* Start the mapping list */
-
-	list = CALLOC (1, sizeof (int16_t) * 3);
-	ERR_ABORT (list);
-	list[0] = priv->child_count; //namespace's index
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send symlink request to all the nodes now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, no node online, "
-			"sending unlink to namespace");
-      
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-	
-	STACK_WIND (frame,
-		    unify_symlink_cbk,
-		    sched_xl,
-		    sched_xl->fops->symlink,
-		    local->name,
-		    &local->loc1);
-
-	return 0;
-}
-
-/**
- * unify_symlink - 
- */
-int32_t
-unify_symlink (call_frame_t *frame,
-	       xlator_t *this,
-	       const char *linkpath,
-	       loc_t *loc)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->name = strdup (linkpath);
-
-	if ((local->name == NULL) || 
-	    (local->loc1.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_symlink_cbk,
-		    NS(this),
-		    NS(this)->fops->symlink,
-		    linkpath,
-		    loc);
-
-	return 0;
-}
-
-
-int32_t 
-unify_rename_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s -> %s): %s", 
-			prev_frame->this->name, 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-      
-	}
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.st_ino = local->st_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_undo_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  struct stat *buf)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-	}
-
-	local->stbuf.st_ino = local->st_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, &local->stbuf);
-	return 0;
-}
-
-int32_t 
-unify_rename_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct stat *buf)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (!S_ISDIR (buf->st_mode))
-				local->stbuf = *buf;
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"child(%s): path(%s -> %s): %s", 
-				prev_frame->this->name, 
-				local->loc1.path, local->loc2.path, 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.st_ino = local->st_ino;
-		if (S_ISDIR (local->loc1.inode->st_mode)) {
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, &local->stbuf);
-			return 0;
-		}
-
-		if (local->op_ret == -1) {
-			/* TODO: check this logic */
-
-			/* Rename failed in storage node, successful on NS, 
-			 * hence, rename back the entries in NS */
-			/* NOTE: this will be done only if the destination 
-			 * doesn't exists, if  the destination exists, the 
-			 * job of correcting NS is left to self-heal
-			 */
-			if (!local->index) {
-				loc_t tmp_oldloc = {
-                                        /* its actual 'newloc->path' */
-					.path = local->loc2.path, 
-					.inode = local->loc1.inode,
-					.parent = local->loc2.parent
-				};
-	
-				loc_t tmp_newloc = {
-					/* Actual 'oldloc->path' */
-					.path = local->loc1.path,
-					.parent = local->loc1.parent
-				};
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"rename succussful on namespace, on "
-					"stroage node failed, reverting back");
-
-				STACK_WIND (frame,
-					    unify_ns_rename_undo_cbk,
-					    NS(this),
-					    NS(this)->fops->rename,
-					    &tmp_oldloc,
-					    &tmp_newloc);
-				return 0;
-			}
-		} else {
-			/* Rename successful on storage nodes */
-
-			int32_t idx = 0;
-			int16_t *tmp_list = NULL;
-			uint64_t tmp_list_int64 = 0;
-			if (local->loc2.inode) {
-				inode_ctx_get (local->loc2.inode, 
-					       this, &tmp_list_int64);
-				list = (int16_t *)(long)tmp_list_int64;
-
-			}
-
-			if (list) {				
-				for (index = 0; list[index] != -1; index++);
-				tmp_list = CALLOC (1, index * 2);
-				memcpy (tmp_list, list, index * 2);
-
-				for (index = 0; list[index] != -1; index++) {
-					/* TODO: Check this logic. */
-					/* If the destination file exists in 
-					 * the same storage node where we sent
-					 * 'rename' call, no need to send 
-					 * unlink 
-					 */
-					for (idx = 0; 
-					     local->list[idx] != -1; idx++) {
-						if (tmp_list[index] == local->list[idx]) {
-							tmp_list[index] = priv->child_count;
-							continue;
-						}
-					}
-	  
-					if (NS(this) != priv->xl_array[tmp_list[index]]) {
-						local->call_count++;
-						callcnt++;
-					}
-				}
-
-				if (local->call_count) {
-					if (callcnt > 1)
-						gf_log (this->name, 
-							GF_LOG_ERROR, 
-							"%s->%s: more (%d) "
-							"subvolumes have the "
-							"newloc entry", 
-							local->loc1.path, 
-							local->loc2.path, 
-							callcnt);
-
-					for (index=0; 
-					     tmp_list[index] != -1; index++) {
-						if (NS(this) != priv->xl_array[tmp_list[index]]) {		    
-							STACK_WIND (frame,
-								    unify_rename_unlink_cbk,
-								    priv->xl_array[tmp_list[index]],
-								    priv->xl_array[tmp_list[index]]->fops->unlink,
-								    &local->loc2);
-							if (!--callcnt)
-								break;
-						}
-					}
-
-					FREE (tmp_list);
-					return 0;
-				}
-				if (tmp_list)
-					FREE (tmp_list);
-			}
-		}
-    
-		/* Need not send 'unlink' to storage node */
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, &local->stbuf);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct stat *buf)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Free local->new_inode */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, buf);
-		return 0;
-	}
-
-	local->stbuf = *buf;
-	local->st_ino = buf->st_ino;
-
-	/* Everything is fine. */
-	if (S_ISDIR (buf->st_mode)) {
-		local->call_count = priv->child_count;
-		for (index=0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_rename_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->rename,
-				    &local->loc1,
-				    &local->loc2);
-		}
-
-		return 0;
-	}
-
-	local->call_count = 0;  
-	/* send rename */
-	list = local->list;
-	for (index=0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			callcnt++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index=0; list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[list[index]]) {
-				STACK_WIND (frame,
-					    unify_rename_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->rename,
-					    &local->loc1,
-					    &local->loc2);
-				if (!--callcnt)
-					break;
-			}
-		}
-	} else {
-		/* file doesn't seem to be present in storage nodes */
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"CRITICAL: source file not in storage node, "
-			"rename successful on namespace :O");
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, -1, EIO, NULL);
-	}
-	return 0;
-}
-
-
-/**
- * unify_rename - One of the tricky function. The deadliest of all :O
- */
-int32_t
-unify_rename (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *oldloc,
-	      loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	if ((local->loc1.path == NULL) || 
-	    (local->loc2.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL);
-		return 0;
-	}
-  
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_rename_cbk,
-		    NS(this),
-		    NS(this)->fops->rename,
-		    oldloc,
-		    newloc);
-	return 0;
-}
-
-/**
- * unify_link_cbk -
- */
-int32_t
-unify_link_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-		struct stat *buf)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret >= 0) 
-		local->stbuf = *buf;
-	local->stbuf.st_ino = local->st_ino;
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf);
-
-	return 0;
-}
-
-/**
- * unify_ns_link_cbk - 
- */
-int32_t
-unify_ns_link_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-		   struct stat *buf)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	int16_t *list = local->list;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send link request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-		return 0;
-	}
-
-	/* Update inode for this entry */
-	local->op_ret = 0;
-	local->st_ino = buf->st_ino;
-
-	/* Send link request to the node now */
-	for (index = 0; list[index] != -1; index++) {
-		char need_break = (list[index+1] == -1);
-		if (priv->xl_array[list[index]] != NS (this)) {
-			STACK_WIND (frame,
-				    unify_link_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->link,
-				    &local->loc1,
-				    &local->loc2);
-		}
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-/**
- * unify_link - 
- */
-int32_t
-unify_link (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *oldloc,
-	    loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (oldloc);
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (newloc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_link_cbk,
-		    NS(this),
-		    NS(this)->fops->link,
-		    oldloc,
-		    newloc);
-
-	return 0;
-}
-
-
-/**
- * unify_checksum_cbk - 
- */
-int32_t
-unify_checksum_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    uint8_t *fchecksum,
-		    uint8_t *dchecksum)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fchecksum, dchecksum);
-
-	return 0;
-}
-
-/**
- * unify_checksum - 
- */
-int32_t
-unify_checksum (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		int32_t flag)
-{
-	STACK_WIND (frame,
-		    unify_checksum_cbk,
-		    NS(this),
-		    NS(this)->fops->checksum,
-		    loc,
-		    flag);
-
-	return 0;
-}
-
-
-/**
- * unify_finodelk_cbk - 
- */
-int
-unify_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_finodelk
- */
-int
-unify_finodelk (call_frame_t *frame, xlator_t *this,
-		fd_t *fd, int cmd, struct flock *flock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_finodelk_cbk,
-		    child, child->fops->finodelk,
-		    fd, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fentrylk_cbk - 
- */
-int
-unify_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_fentrylk
- */
-int
-unify_fentrylk (call_frame_t *frame, xlator_t *this,
-		fd_t *fd, const char *basename,
-		entrylk_cmd cmd, entrylk_type type)
-		
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fentrylk_cbk,
-		    child, child->fops->fentrylk,
-		    fd, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fxattrop_cbk - 
- */
-int
-unify_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_fxattrop
- */
-int
-unify_fxattrop (call_frame_t *frame, xlator_t *this,
-		fd_t *fd, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fxattrop_cbk,
-		    child, child->fops->fxattrop,
-		    fd, optype, xattr);
-
-	return 0;
-}
-
-
-/**
- * unify_inodelk_cbk - 
- */
-int
-unify_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_inodelk
- */
-int
-unify_inodelk (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, int cmd, struct flock *flock)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_inodelk_cbk,
-		    child, child->fops->inodelk,
-		    loc, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_entrylk_cbk - 
- */
-int
-unify_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_entrylk
- */
-int
-unify_entrylk (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, const char *basename,
-	       entrylk_cmd cmd, entrylk_type type)
-		
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_entrylk_cbk,
-		    child, child->fops->entrylk,
-		    loc, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_xattrop_cbk - 
- */
-int
-unify_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_xattrop
- */
-int
-unify_xattrop (call_frame_t *frame, xlator_t *this,
-		loc_t *loc, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_xattrop_cbk,
-		    child, child->fops->xattrop,
-		    loc, optype, xattr);
-
-	return 0;
-}
-
-
-/**
- * notify
- */
-int32_t
-notify (xlator_t *this,
-        int32_t event,
-        void *data,
-        ...)
-{
-	unify_private_t *priv = this->private;
-	struct sched_ops *sched = NULL;
-
-	if (!priv) {
-		return 0;
-	}
-
-	sched = priv->sched_ops;    
-	if (!sched) {
-		gf_log (this->name, GF_LOG_CRITICAL, "No scheduler :O");
-		raise (SIGTERM);
-		return 0;
-	}
-	if (priv->namespace == data) {
-		if (event == GF_EVENT_CHILD_UP) {
-			sched->notify (this, event, data);
-		}
-		return 0;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		/* Call scheduler's update () to enable it for scheduling */
-		sched->notify (this, event, data);
-	
-		LOCK (&priv->lock);
-		{
-			/* Increment the inode's generation, which is 
-			   used for self_heal */
-			++priv->inode_generation;
-			++priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (!priv->is_up) {
-			default_notify (this, event, data);
-			priv->is_up = 1;
-		}
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		/* Call scheduler's update () to disable the child node 
-		 * for scheduling
-		 */
-		sched->notify (this, event, data);
-		LOCK (&priv->lock);
-		{
-			--priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (priv->num_child_up == 0) {
-			/* Send CHILD_DOWN to upper layer */
-			default_notify (this, event, data);
-			priv->is_up = 0;
-		}
-	}
-	break;
-
-	default:
-	{
-		default_notify (this, event, data);
-	}
-	break;
-	}
-
-	return 0;
-}
-
-/** 
- * init - This function is called first in the xlator, while initializing.
- *   All the config file options are checked and appropriate flags are set.
- *
- * @this - 
- */
-int32_t 
-init (xlator_t *this)
-{
-	int32_t          ret       = 0;
-	int32_t          count     = 0;
-	data_t          *scheduler = NULL;
-	data_t          *data      = NULL;
-	xlator_t        *ns_xl     = NULL;
-	xlator_list_t   *trav      = NULL;
-	xlator_list_t   *xlparent  = NULL;
-	xlator_list_t   *parent    = NULL;
-	unify_private_t *_private  = NULL; 
-
-	/* Check for number of child nodes, if there is no child nodes, exit */
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"No child nodes specified. check \"subvolumes \" "
-			"option in volfile");
-		return -1;
-	}
-
-  	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	/* Check for 'scheduler' in volume */
-	scheduler = dict_get (this->options, "scheduler");
-	if (!scheduler) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"\"option scheduler <x>\" is missing in volfile");
-		return -1;
-	}
-
-	/* Setting "option namespace <node>" */
-	data = dict_get (this->options, "namespace");
-	if(!data) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace option not specified, Exiting");
-		return -1;
-	}
-	/* Search namespace in the child node, if found, exit */
-	trav = this->children;
-	while (trav) {
-		if (strcmp (trav->xlator->name, data->data) == 0)
-			break;
-		trav = trav->next;
-	}
-	if (trav) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node used as a subvolume, Exiting");
-		return -1;
-	}
-	
-	/* Search for the namespace node, if found, continue */
-	ns_xl = this->next;
-	while (ns_xl) {
-		if (strcmp (ns_xl->name, data->data) == 0)
-			break;
-		ns_xl = ns_xl->next;
-	}
-	if (!ns_xl) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node not found in volfile, Exiting");
-		return -1;
-	}
-	
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"namespace node specified as %s", data->data);
-	
-	_private = CALLOC (1, sizeof (*_private));
-	ERR_ABORT (_private);
-	_private->sched_ops = get_scheduler (this, scheduler->data);
-	if (!_private->sched_ops) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"Error while loading scheduler. Exiting");
-		FREE (_private);
-		return -1;
-	}
-	
-	if (ns_xl->parents) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Namespace node should not be a child of any other node. Exiting");
-		FREE (_private);
-		return -1;
-	}
-
-	_private->namespace = ns_xl;
-	
-	/* update _private structure */
-	{
-		count = 0;
-		trav = this->children;
-		/* Get the number of child count */
-		while (trav) {
-			count++;
-			trav = trav->next;
-		}
-		
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"Child node count is %d", count);    
-
-		_private->child_count = count;
-		if (count == 1) {
-			/* TODO: Should I error out here? */
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"WARNING: You have defined only one "
-				"\"subvolumes\" for unify volume. It may not "
-				"be the desired config, review your volume "
-				"volfile. If this is how you are testing it,"
-				" you may hit some performance penalty");
-		}
-		
-		_private->xl_array = CALLOC (1, 
-					     sizeof (xlator_t) * (count + 1));
-		ERR_ABORT (_private->xl_array);
-		
-		count = 0;
-		trav = this->children;
-		while (trav) {
-			_private->xl_array[count++] = trav->xlator;
-			trav = trav->next;
-		}
-		_private->xl_array[count] = _private->namespace;
-		
-		/* self-heal part, start with generation '1' */
-		_private->inode_generation = 1; 
-                /* Because, Foreground part is tested well */
-		_private->self_heal = ZR_UNIFY_FG_SELF_HEAL; 
-		data = dict_get (this->options, "self-heal");
-		if (data) {
-			if (strcasecmp (data->data, "off") == 0) 
-				_private->self_heal = ZR_UNIFY_SELF_HEAL_OFF;
-
-			if (strcasecmp (data->data, "foreground") == 0)
-				_private->self_heal = ZR_UNIFY_FG_SELF_HEAL;
-
-			if (strcasecmp (data->data, "background") == 0)
-				_private->self_heal = ZR_UNIFY_BG_SELF_HEAL;
-		}
-    
-		/* optimist - ask bulde for more about it */
-		data = dict_get (this->options, "optimist");
-		if (data) {
-			if (gf_string2boolean (data->data, 
-					       &_private->optimist) == -1) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"optimist excepts only boolean "
-					"options");
-			}
-		}
-
-		LOCK_INIT (&_private->lock);
-	}
-
-	/* Now that everything is fine. */
-	this->private = (void *)_private;
-	{
-		/* Initialize scheduler, if everything else is successful */
-		ret = _private->sched_ops->init (this); 
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_CRITICAL,
-				"Initializing scheduler failed, Exiting");
-			FREE (_private);
-			return -1;
-		}
-
-		ret = 0;
-
-		/* This section is required because some fops may look 
-		 * for 'xl->parent' variable 
-		 */
-		xlparent = CALLOC (1, sizeof (*xlparent));
-		xlparent->xlator = this;
-		if (!ns_xl->parents) {
-			ns_xl->parents = xlparent;
-		} else {
-			parent = ns_xl->parents;
-			while (parent->next)
-				parent = parent->next;
-			parent->next = xlparent;
-		}
-		/* Initialize the namespace volume */
-		if (!ns_xl->ready) {
-			ret = xlator_tree_init (ns_xl);
-			if (ret) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"initializing namespace node failed, "
-					"Exiting");
-			FREE (_private);
-			return -1;
-			}
-		}
-	}
-
-	/* Tell namespace node that init is done */
-	ns_xl->notify (ns_xl, GF_EVENT_PARENT_UP, this);
-
-	return 0;
-}
-
-/** 
- * fini  - Free all the allocated memory 
- */
-void
-fini (xlator_t *this)
-{
-	unify_private_t *priv = this->private;
-	priv->sched_ops->fini (this);
-	this->private = NULL;
-	LOCK_DESTROY (&priv->lock);
-	FREE (priv->xl_array);
-	FREE (priv);
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.stat        = unify_stat,
-	.chmod       = unify_chmod,
-	.readlink    = unify_readlink,
-	.mknod       = unify_mknod,
-	.mkdir       = unify_mkdir,
-	.unlink      = unify_unlink,
-	.rmdir       = unify_rmdir,
-	.symlink     = unify_symlink,
-	.rename      = unify_rename,
-	.link        = unify_link,
-	.chown       = unify_chown,
-	.truncate    = unify_truncate,
-	.create      = unify_create,
-	.open        = unify_open,
-	.readv       = unify_readv,
-	.writev      = unify_writev,
-	.statfs      = unify_statfs,
-	.flush       = unify_flush,
-	.fsync       = unify_fsync,
-	.setxattr    = unify_setxattr,
-	.getxattr    = unify_getxattr,
-	.removexattr = unify_removexattr,
-	.opendir     = unify_opendir,
-	.readdir     = unify_readdir,
-	.fsyncdir    = unify_fsyncdir,
-	.access      = unify_access,
-	.ftruncate   = unify_ftruncate,
-	.fstat       = unify_fstat,
-	.lk          = unify_lk,
-	.fchown      = unify_fchown,
-	.fchmod      = unify_fchmod,
-	.utimens     = unify_utimens,
-	.lookup      = unify_lookup,
-	.getdents    = unify_getdents,
-	.checksum    = unify_checksum,
-	.inodelk     = unify_inodelk,
-	.finodelk    = unify_finodelk,
-	.entrylk     = unify_entrylk,
-	.fentrylk    = unify_fentrylk,
-	.xattrop     = unify_xattrop,
-	.fxattrop    = unify_fxattrop
-};
-
-struct xlator_mops mops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-	{ .key   = { "namespace" },  
-	  .type  = GF_OPTION_TYPE_XLATOR 
-	},
-	{ .key   = { "scheduler" },  
-	  .value = { "alu", "rr", "random", "nufa", "switch" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	{ .key   = {"self-heal"},  
-	  .value = { "foreground", "background", "off" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	/* TODO: remove it some time later */
-	{ .key   = {"optimist"},  
-	  .type  = GF_OPTION_TYPE_BOOL 
-	},
-
-	{ .key   = {NULL} },
-};
diff --git a/xlators/cluster/unify/src/unify.h b/xlators/cluster/unify/src/unify.h
deleted file mode 100644
index bc18dc53f..000000000
--- a/xlators/cluster/unify/src/unify.h
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
-  Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef _UNIFY_H
-#define _UNIFY_H
-
-#include "scheduler.h"
-#include "list.h"
-
-#define MAX_DIR_ENTRY_STRING     (32 * 1024)
-
-#define ZR_UNIFY_SELF_HEAL_OFF 0
-#define ZR_UNIFY_FG_SELF_HEAL  1
-#define ZR_UNIFY_BG_SELF_HEAL  2
-
-/* Sometimes one should use completely random numbers.. its good :p */
-#define UNIFY_SELF_HEAL_GETDENTS_COUNT 1024
-
-#define NS(xl)          (((unify_private_t *)xl->private)->namespace)
-
-/* This is used to allocate memory for local structure */
-#define INIT_LOCAL(fr, loc)                   \
-do {                                          \
-  loc = CALLOC (1, sizeof (unify_local_t));   \
-  ERR_ABORT (loc);			      \
-  if (!loc) {                                 \
-    STACK_UNWIND (fr, -1, ENOMEM);            \
-    return 0;                                 \
-  }                                           \
-  fr->local = loc;                            \
-  loc->op_ret = -1;                           \
-  loc->op_errno = ENOENT;                     \
-} while (0)
-
-
-
-struct unify_private {
-	/* Update this structure depending on requirement */
-	void *scheduler;               /* THIS SHOULD BE THE FIRST VARIABLE, 
-					  if xlator is using scheduler */
-	struct sched_ops *sched_ops;   /* Scheduler options  */
-	xlator_t *namespace;           /* ptr to namespace xlator */
-	xlator_t **xl_array;
-	gf_boolean_t optimist;
-	int16_t child_count;
-	int16_t num_child_up;
-	uint8_t self_heal;
-	uint8_t is_up;
-	uint64_t inode_generation;
-	gf_lock_t lock;
-};
-typedef struct unify_private unify_private_t;
-
-struct unify_self_heal_struct {
-	uint8_t dir_checksum[ZR_FILENAME_MAX];
-	uint8_t ns_dir_checksum[ZR_FILENAME_MAX];
-	uint8_t file_checksum[ZR_FILENAME_MAX];
-	uint8_t ns_file_checksum[ZR_FILENAME_MAX];
-	off_t *offset_list;
-	int   *count_list;
-	dir_entry_t **entry_list;
-};
-
-
-struct _unify_local_t {
-	int32_t call_count;
-	int32_t op_ret;
-	int32_t op_errno;
-	mode_t mode;
-	off_t offset;
-	dev_t dev;
-	uid_t uid;
-	gid_t gid;
-	int32_t flags;
-	int32_t entry_count;
-	int32_t count;    // dir_entry_t count;
-	fd_t *fd;
-	struct stat stbuf;
-	struct statvfs statvfs_buf;
-	struct timespec tv[2];
-	char *name;
-	int32_t revalidate;
-
-	ino_t st_ino;
-	nlink_t st_nlink;
-  
-	dict_t *dict;
-
-	int16_t *list;
-	int16_t *new_list; /* Used only in case of rename */
-	int16_t index;
-
-	int32_t failed;
-	int32_t return_eio;  /* Used in case of different st-mode 
-				present for a given path */
-
-	uint64_t inode_generation; /* used to store the per directory 
-				    * inode_generation. Got from inode's ctx 
-				    * of directory inodes
-				    */
-
-	struct unify_self_heal_struct *sh_struct;
-	loc_t loc1, loc2;
-};
-typedef struct _unify_local_t unify_local_t;
-
-int32_t zr_unify_self_heal (call_frame_t *frame,
-			    xlator_t *this,
-			    unify_local_t *local);
-
-#endif /* _UNIFY_H */
diff --git a/xlators/debug/Makefile.am b/xlators/debug/Makefile.am
index 16cf893a1..b655554ef 100644
--- a/xlators/debug/Makefile.am
+++ b/xlators/debug/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = trace error-gen
+SUBDIRS = trace error-gen io-stats
 
 CLEANFILES = 
diff --git a/xlators/debug/error-gen/src/Makefile.am b/xlators/debug/error-gen/src/Makefile.am
index 1bd7f332c..5075c59a8 100644
--- a/xlators/debug/error-gen/src/Makefile.am
+++ b/xlators/debug/error-gen/src/Makefile.am
@@ -2,13 +2,16 @@
 xlator_LTLIBRARIES = error-gen.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-error_gen_la_LDFLAGS = -module -avoidversion
+error_gen_la_LDFLAGS = -module -avoid-version
 
 error_gen_la_SOURCES = error-gen.c
 error_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = error-gen.h error-gen-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/error-gen/src/error-gen-mem-types.h b/xlators/debug/error-gen/src/error-gen-mem-types.h
new file mode 100644
index 000000000..f02280535
--- /dev/null
+++ b/xlators/debug/error-gen/src/error-gen-mem-types.h
@@ -0,0 +1,20 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __ERROR_GEN_MEM_TYPES_H__
+#define __ERROR_GEN_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_error_gen_mem_types_ {
+        gf_error_gen_mt_eg_t = gf_common_mt_end + 1,
+        gf_error_gen_mt_end
+};
+#endif
diff --git a/xlators/debug/error-gen/src/error-gen.c b/xlators/debug/error-gen/src/error-gen.c
index 9c0b3253e..ec0874b35 100644
--- a/xlators/debug/error-gen/src/error-gen.c
+++ b/xlators/debug/error-gen/src/error-gen.c
@@ -1,1727 +1,2163 @@
 /*
-  Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
 #include "xlator.h"
+#include "error-gen.h"
+#include "statedump.h"
+
+sys_error_t error_no_list[] = {
+        [GF_FOP_LOOKUP]            = { .error_no_count = 4,
+                                    .error_no = {ENOENT,ENOTDIR,
+                                                 ENAMETOOLONG,EAGAIN}},
+        [GF_FOP_STAT]              = { .error_no_count = 7,
+                                    .error_no = {EACCES,EBADF,EFAULT,
+                                                 ENAMETOOLONG,ENOENT,
+                                                 ENOMEM,ENOTDIR}},
+        [GF_FOP_READLINK]          = { .error_no_count = 8,
+                                    .error_no = {EACCES,EFAULT,EINVAL,EIO,
+                                                 ENAMETOOLONG,ENOENT,ENOMEM,
+                                                 ENOTDIR}},
+        [GF_FOP_MKNOD]             = { .error_no_count = 11,
+                                    .error_no = {EACCES,EEXIST,EFAULT,
+                                                 EINVAL,ENAMETOOLONG,
+                                                 ENOENT,ENOMEM,ENOSPC,
+                                                 ENOTDIR,EPERM,EROFS}},
+        [GF_FOP_MKDIR]             = { .error_no_count = 10,
+                                    .error_no = {EACCES,EEXIST,EFAULT,
+                                                 ENAMETOOLONG,ENOENT,
+                                                 ENOMEM,ENOSPC,ENOTDIR,
+						 EPERM,EROFS}},
+        [GF_FOP_UNLINK]            = { .error_no_count = 10,
+                                    .error_no = {EACCES,EBUSY,EFAULT,EIO,
+                                                 EISDIR,ENAMETOOLONG,
+                                                 ENOENT,ENOMEM,ENOTDIR,
+                                                 EPERM,EROFS}},
+        [GF_FOP_RMDIR]             = { .error_no_count = 8,
+                                    .error_no = {EACCES,EBUSY,EFAULT,
+                                                 ENOMEM,ENOTDIR,ENOTEMPTY,
+                                                 EPERM,EROFS}},
+        [GF_FOP_SYMLINK]           = { .error_no_count = 11,
+                                    .error_no = {EACCES,EEXIST,EFAULT,EIO,
+                                                 ENAMETOOLONG,ENOENT,ENOMEM,
+                                                 ENOSPC,ENOTDIR,EPERM,
+                                                 EROFS}},
+        [GF_FOP_RENAME]            = { .error_no_count = 13,
+                                    .error_no = {EACCES,EBUSY,EFAULT,
+                                                 EINVAL,EISDIR,EMLINK,
+                                                 ENAMETOOLONG,ENOENT,ENOMEM,
+                                                 ENOSPC,ENOTDIR,EEXIST,
+                                                 EXDEV}},
+        [GF_FOP_LINK]              = { .error_no_count = 13,
+                                    .error_no = {EACCES,EFAULT,EEXIST,EIO,
+                                                 EMLINK,ENAMETOOLONG,
+                                                 ENOENT,ENOMEM,ENOSPC,
+                                                 ENOTDIR,EPERM,EROFS,
+                                                 EXDEV}},
+        [GF_FOP_TRUNCATE]          = { .error_no_count = 10,
+                                    .error_no = {EACCES,EFAULT,EFBIG,
+                                                 EINTR,EINVAL,EIO,EISDIR,
+                                                 ENAMETOOLONG,ENOENT,
+                                                 EISDIR}},
+        [GF_FOP_CREATE]            = {.error_no_count = 10,
+                                   .error_no = {EACCES,EEXIST,EFAULT,
+                                                EISDIR,EMFILE,ENAMETOOLONG,
+                                                ENFILE,ENODEV,ENOENT,
+                                                ENODEV}},
+        [GF_FOP_OPEN]              = { .error_no_count = 10,
+                                    .error_no = {EACCES,EEXIST,EFAULT,
+                                                 EISDIR,EMFILE,
+                                                 ENAMETOOLONG,ENFILE,
+                                                 ENODEV,ENOENT,ENOMEM}},
+        [GF_FOP_READ]              = { .error_no_count = 5,
+                                    .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
+                                                 ENAMETOOLONG}},
+        [GF_FOP_WRITE]             = { .error_no_count = 7,
+                                    .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
+                                                 ENAMETOOLONG,ENOSPC,
+						 GF_ERROR_SHORT_WRITE}},
+        [GF_FOP_STATFS]            = {.error_no_count = 10,
+                                   .error_no = {EACCES,EBADF,EFAULT,EINTR,
+                                                EIO,ENAMETOOLONG,ENOENT,
+                                                ENOMEM,ENOSYS,ENOTDIR}},
+        [GF_FOP_FLUSH]             = { .error_no_count = 5,
+                                    .error_no = {EACCES,EFAULT,
+                                                 ENAMETOOLONG,ENOSYS,
+                                                 ENOENT}},
+        [GF_FOP_FSYNC]             = { .error_no_count = 4,
+                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
+        [GF_FOP_SETXATTR]          = { .error_no_count = 4,
+                                    .error_no = {EACCES,EBADF,EINTR,
+                                                 ENAMETOOLONG}},
+        [GF_FOP_GETXATTR]          = { .error_no_count = 4,
+                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
+                                                 EINTR}},
+        [GF_FOP_REMOVEXATTR]       = { .error_no_count = 4,
+                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
+                                                 EINTR}},
+        [GF_FOP_FSETXATTR]          = { .error_no_count = 4,
+                                        .error_no = {EACCES,EBADF,EINTR,
+                                                     ENAMETOOLONG}},
+        [GF_FOP_FGETXATTR]          = { .error_no_count = 4,
+                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
+                                                     EINTR}},
+        [GF_FOP_FREMOVEXATTR]       = { .error_no_count = 4,
+                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
+                                                     EINTR}},
+        [GF_FOP_OPENDIR]           = { .error_no_count = 8,
+                                    .error_no = {EACCES,EEXIST,EFAULT,
+                                                 EISDIR,EMFILE,
+                                                 ENAMETOOLONG,ENFILE,
+                                                 ENODEV}},
+        [GF_FOP_READDIR]           = { .error_no_count = 5,
+                                    .error_no = {EINVAL,EACCES,EBADF,
+                                                 EMFILE,ENOENT}},
+        [GF_FOP_READDIRP]          = { .error_no_count = 5,
+                                    .error_no = {EINVAL,EACCES,EBADF,
+                                                 EMFILE,ENOENT}},
+        [GF_FOP_FSYNCDIR]          = { .error_no_count = 4,
+                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
+        [GF_FOP_ACCESS]            = { .error_no_count = 8,
+                                    .error_no = {EACCES,ENAMETOOLONG,
+                                                 ENOENT,ENOTDIR,EROFS,
+                                                 EFAULT,EINVAL,EIO}},
+        [GF_FOP_FTRUNCATE]         = { .error_no_count = 9,
+                                    .error_no = {EACCES,EFAULT,EFBIG,
+                                                 EINTR,EINVAL,EIO,EISDIR,
+                                                 ENAMETOOLONG,ENOENT}},
+        [GF_FOP_FSTAT]             = { .error_no_count = 7,
+                                    .error_no = {EACCES,EBADF,EFAULT,
+                                                 ENAMETOOLONG,ENOENT,
+                                                 ENOMEM,ENOTDIR}},
+        [GF_FOP_LK]                = { .error_no_count = 4,
+                                    .error_no = {EACCES,EFAULT,ENOENT,
+                                                 EINTR}},
+        [GF_FOP_XATTROP]           = { .error_no_count = 5,
+                                    .error_no = {EACCES,EFAULT,
+                                                 ENAMETOOLONG,ENOSYS,
+                                                 ENOENT}},
+        [GF_FOP_FXATTROP]          = { .error_no_count = 4,
+                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
+        [GF_FOP_INODELK]           = { .error_no_count = 4,
+                                    .error_no = {EACCES,EBADF,EINTR,
+                                                 ENAMETOOLONG}},
+        [GF_FOP_FINODELK]          = { .error_no_count = 4,
+                                    .error_no = {EACCES,EBADF,EINTR,
+                                                 ENAMETOOLONG}},
+        [GF_FOP_ENTRYLK]           = { .error_no_count = 4,
+                                    .error_no = {EACCES,EBADF,
+                                                 ENAMETOOLONG,EINTR}},
+        [GF_FOP_FENTRYLK]          = { .error_no_count = 10,
+                                    .error_no = {EACCES,EEXIST,EFAULT,
+                                                 EISDIR,EMFILE,
+                                                 ENAMETOOLONG,ENFILE,
+                                                 ENODEV,ENOENT,ENOMEM}},
+        [GF_FOP_SETATTR]           = {.error_no_count = 11,
+                                    .error_no = {EACCES,EFAULT,EIO,
+                                                 ENAMETOOLONG,ENOENT,
+                                                 ENOMEM,ENOTDIR,EPERM,
+                                                 EROFS,EBADF,EIO}},
+        [GF_FOP_FSETATTR]          = { .error_no_count = 11,
+                                    .error_no = {EACCES,EFAULT,EIO,
+                                                 ENAMETOOLONG,ENOENT,
+                                                 ENOMEM,ENOTDIR,EPERM,
+                                                 EROFS,EBADF,EIO}},
+        [GF_FOP_GETSPEC]           = { .error_no_count = 4,
+                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
+                                                 EINTR}}
+};
 
-typedef struct {
-	int op_count;
-} eg_t;
-
-int error_gen (xlator_t *this)
-{
-	eg_t *egp = NULL;
-	int count = 0;
-	egp = this->private;
-	count = ++egp->op_count;
-	if((count % 10) == 0) {
-		count = count / 10;
-		if ((count % 2) == 0)
-			return ENOTCONN;
-		else
-			return EIO;
-	}
-	return 0;
+int
+generate_rand_no (int op_no)
+{
+        int             rand_no = 0;
+
+        if (op_no < GF_FOP_MAXVALUE)
+                rand_no = rand () % error_no_list[op_no].error_no_count;
+        return rand_no;
 }
 
-static int32_t
-error_gen_lookup_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-		      struct stat *buf,
-		      dict_t *dict)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      inode,
-		      buf,
-		      dict);
-	return 0;
+int
+conv_errno_to_int (char **error_no)
+{
+        if (!strcmp ((*error_no), "ENOENT"))
+                return ENOENT;
+        else if (!strcmp ((*error_no), "ENOTDIR"))
+                return ENOTDIR;
+        else if (!strcmp ((*error_no), "ENAMETOOLONG"))
+                return ENAMETOOLONG;
+        else if (!strcmp ((*error_no), "EACCES"))
+                return EACCES;
+        else if (!strcmp ((*error_no), "EBADF"))
+                return EBADF;
+        else if (!strcmp ((*error_no), "EFAULT"))
+                return EFAULT;
+        else if (!strcmp ((*error_no), "ENOMEM"))
+                return ENOMEM;
+        else if (!strcmp ((*error_no), "EINVAL"))
+                return EINVAL;
+        else if (!strcmp ((*error_no), "EIO"))
+                return EIO;
+        else if (!strcmp ((*error_no), "EEXIST"))
+                return EEXIST;
+        else if (!strcmp ((*error_no), "ENOSPC"))
+                return ENOSPC;
+        else if (!strcmp ((*error_no), "EPERM"))
+                return EPERM;
+        else if (!strcmp ((*error_no), "EROFS"))
+                return EROFS;
+        else if (!strcmp ((*error_no), "EBUSY"))
+                return EBUSY;
+        else if (!strcmp ((*error_no), "EISDIR"))
+                return EISDIR;
+        else if (!strcmp ((*error_no), "ENOTEMPTY"))
+                return ENOTEMPTY;
+        else if (!strcmp ((*error_no), "EMLINK"))
+                return EMLINK;
+        else if (!strcmp ((*error_no), "ENODEV"))
+                return ENODEV;
+        else if (!strcmp ((*error_no), "EXDEV"))
+                return EXDEV;
+        else if (!strcmp ((*error_no), "EMFILE"))
+                return EMFILE;
+        else if (!strcmp ((*error_no), "ENFILE"))
+                return ENFILE;
+        else if (!strcmp ((*error_no), "ENOSYS"))
+                return ENOSYS;
+        else if (!strcmp ((*error_no), "EINTR"))
+                return EINTR;
+        else if (!strcmp ((*error_no), "EFBIG"))
+                return EFBIG;
+	else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
+		return GF_ERROR_SHORT_WRITE;
+        else
+                return EAGAIN;
 }
 
-int32_t
-error_gen_lookup (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  dict_t *xattr_req)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    error_gen_lookup_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lookup,
-		    loc,
-		    xattr_req);
-	return 0;
+int
+get_fop_int (char **op_no_str)
+{
+        if (!strcmp ((*op_no_str), "lookup"))
+                return GF_FOP_LOOKUP;
+        else if (!strcmp ((*op_no_str), "stat"))
+                return GF_FOP_STAT;
+        else if (!strcmp ((*op_no_str), "readlink"))
+                return GF_FOP_READLINK;
+        else if (!strcmp ((*op_no_str), "mknod"))
+                return GF_FOP_MKNOD;
+        else if (!strcmp ((*op_no_str), "mkdir"))
+                return GF_FOP_MKDIR;
+        else if (!strcmp ((*op_no_str), "unlink"))
+                return GF_FOP_UNLINK;
+        else if (!strcmp ((*op_no_str), "rmdir"))
+                return GF_FOP_RMDIR;
+        else if (!strcmp ((*op_no_str), "symlink"))
+                return GF_FOP_SYMLINK;
+        else if (!strcmp ((*op_no_str), "rename"))
+                return GF_FOP_RENAME;
+        else if (!strcmp ((*op_no_str), "link"))
+                return GF_FOP_LINK;
+        else if (!strcmp ((*op_no_str), "truncate"))
+                return GF_FOP_TRUNCATE;
+        else if (!strcmp ((*op_no_str), "create"))
+                return GF_FOP_CREATE;
+        else if (!strcmp ((*op_no_str), "open"))
+                return GF_FOP_OPEN;
+        else if (!strcmp ((*op_no_str), "readv"))
+                return GF_FOP_READ;
+        else if (!strcmp ((*op_no_str), "writev"))
+                return GF_FOP_WRITE;
+        else if (!strcmp ((*op_no_str), "statfs"))
+                return GF_FOP_STATFS;
+        else if (!strcmp ((*op_no_str), "flush"))
+                return GF_FOP_FLUSH;
+        else if (!strcmp ((*op_no_str), "fsync"))
+                return GF_FOP_FSYNC;
+        else if (!strcmp ((*op_no_str), "setxattr"))
+                return GF_FOP_SETXATTR;
+        else if (!strcmp ((*op_no_str), "getxattr"))
+                return GF_FOP_GETXATTR;
+        else if (!strcmp ((*op_no_str), "removexattr"))
+                return GF_FOP_REMOVEXATTR;
+        else if (!strcmp ((*op_no_str), "fsetxattr"))
+                return GF_FOP_FSETXATTR;
+        else if (!strcmp ((*op_no_str), "fgetxattr"))
+                return GF_FOP_FGETXATTR;
+        else if (!strcmp ((*op_no_str), "fremovexattr"))
+                return GF_FOP_FREMOVEXATTR;
+        else if (!strcmp ((*op_no_str), "opendir"))
+                return GF_FOP_OPENDIR;
+        else if (!strcmp ((*op_no_str), "readdir"))
+                return GF_FOP_READDIR;
+        else if (!strcmp ((*op_no_str), "readdirp"))
+                return GF_FOP_READDIRP;
+        else if (!strcmp ((*op_no_str), "fsyncdir"))
+                return GF_FOP_FSYNCDIR;
+        else if (!strcmp ((*op_no_str), "access"))
+                return GF_FOP_ACCESS;
+        else if (!strcmp ((*op_no_str), "ftruncate"))
+                return GF_FOP_FTRUNCATE;
+        else if (!strcmp ((*op_no_str), "fstat"))
+                return GF_FOP_FSTAT;
+        else if (!strcmp ((*op_no_str), "lk"))
+                return GF_FOP_LK;
+        else if (!strcmp ((*op_no_str), "xattrop"))
+                return GF_FOP_XATTROP;
+        else if (!strcmp ((*op_no_str), "fxattrop"))
+                return GF_FOP_FXATTROP;
+        else if (!strcmp ((*op_no_str), "inodelk"))
+                return GF_FOP_INODELK;
+        else if (!strcmp ((*op_no_str), "finodelk"))
+                return GF_FOP_FINODELK;
+        else if (!strcmp ((*op_no_str), "etrylk"))
+                return GF_FOP_ENTRYLK;
+        else if (!strcmp ((*op_no_str), "fentrylk"))
+                return GF_FOP_FENTRYLK;
+        else if (!strcmp ((*op_no_str), "setattr"))
+                return GF_FOP_SETATTR;
+        else if (!strcmp ((*op_no_str), "fsetattr"))
+                return GF_FOP_FSETATTR;
+        else if (!strcmp ((*op_no_str), "getspec"))
+                return GF_FOP_GETSPEC;
+	else
+                return -1;
 }
 
+int
+error_gen (xlator_t *this, int op_no)
+{
+        eg_t             *egp = NULL;
+        int              count = 0;
+        int              failure_iter_no = GF_FAILURE_DEFAULT;
+        char             *error_no = NULL;
+        int              rand_no = 0;
+        int              ret = 0;
+
+        egp = this->private;
+
+        LOCK (&egp->lock);
+        {
+                count = ++egp->op_count;
+                failure_iter_no = egp->failure_iter_no;
+                error_no = egp->error_no;
+        }
+        UNLOCK (&egp->lock);
+
+        if((count % failure_iter_no) == 0) {
+                LOCK (&egp->lock);
+                {
+                        egp->op_count = 0;
+                }
+                UNLOCK (&egp->lock);
+
+                if (error_no)
+                        ret = conv_errno_to_int (&error_no);
+                else {
+
+                        rand_no = generate_rand_no (op_no);
+                        if (op_no >= GF_FOP_MAXVALUE)
+                                op_no = 0;
+                        if (rand_no >= error_no_list[op_no].error_no_count)
+                                rand_no = 0;
+                        ret = error_no_list[op_no].error_no[rand_no];
+                }
+                if (egp->random_failure == _gf_true)
+                        egp->failure_iter_no = 3 + (rand () % GF_UNIVERSAL_ANSWER);
+        }
+        return ret;
+}
 
-int32_t
-error_gen_forget (xlator_t *this,
-		  inode_t *inode)
+
+int
+error_gen_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		      int32_t op_ret, int32_t op_errno, inode_t *inode,
+		      struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-	return 0;
+	STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode,
+                             buf, xdata, postparent);
+        return 0;
 }
 
-int32_t
-error_gen_stat_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
-}
 
-int32_t
-error_gen_stat (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc)
+int
+error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_LOOKUP];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_LOOKUP);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                                     NULL);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_stat_cbk,
+
+	STACK_WIND (frame, error_gen_lookup_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->stat,
-		    loc);
-	return 0;
+		    FIRST_CHILD(this)->fops->lookup,
+		    loc, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_chmod_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
+
+int
+error_gen_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_chmod (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 mode_t mode)
+int
+error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_STAT];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_STAT);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_chmod_cbk,
+
+	STACK_WIND (frame, error_gen_stat_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->chmod,
-		    loc,
-		    mode);
-	return 0;
+		    FIRST_CHILD(this)->fops->stat,
+		    loc, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_fchmod_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
+int
+error_gen_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_fchmod (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  mode_t mode)
+
+int
+error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    error_gen_fchmod_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fchmod,
-		    fd,
-		    mode);
-	return 0;
-}
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
 
-int32_t
-error_gen_chown_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
-}
+        egp = this->private;
+        enable = egp->enable[GF_FOP_SETATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_SETATTR);
 
-int32_t
-error_gen_chown (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 uid_t uid,
-		 gid_t gid)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_chown_cbk,
+
+	STACK_WIND (frame, error_gen_setattr_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->chown,
-		    loc,
-		    uid,
-		    gid);
-	return 0;
+		    FIRST_CHILD(this)->fops->setattr,
+		    loc, stbuf, valid, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_fchown_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
-}
 
-int32_t
-error_gen_fchown (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  uid_t uid,
-		  gid_t gid)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
+int
+error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+	int             op_errno = 0;
+        eg_t            *egp = NULL;
+        int             enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FSETATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FSETATTR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_fchown_cbk,
+
+	STACK_WIND (frame, error_gen_setattr_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fchown,
-		    fd,
-		    uid,
-		    gid);
-	return 0;
+		    FIRST_CHILD(this)->fops->fsetattr,
+		    fd, stbuf, valid, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_truncate_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
+
+int
+error_gen_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			int32_t op_ret, int32_t op_errno,
+			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
+                             prebuf, postbuf, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_truncate (call_frame_t *frame,
-		    xlator_t *this,
-		    loc_t *loc,
-		    off_t offset)
+
+int
+error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    off_t offset, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_TRUNCATE];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_TRUNCATE);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (truncate, frame, -1, op_errno,
+                                     NULL, NULL, xdata);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_truncate_cbk,
+
+	STACK_WIND (frame, error_gen_truncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->truncate,
-		    loc,
-		    offset);
-	return 0;
+		    loc, offset, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_ftruncate_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
+
+int
+error_gen_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
+                             prebuf, postbuf, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_ftruncate (call_frame_t *frame,
-		     xlator_t *this,
-		     fd_t *fd,
-		     off_t offset)
+
+int
+error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		     off_t offset, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp =NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FTRUNCATE];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FTRUNCATE);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno,
+                                     NULL, NULL, xdata);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_ftruncate_cbk,
+
+	STACK_WIND (frame, error_gen_ftruncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd,
-		    offset);
-	return 0;
-}
-
-int32_t
-error_gen_utimens_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
+		    fd, offset, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_utimens (call_frame_t *frame,
-		   xlator_t *this,
-		   loc_t *loc,
-		   struct timespec tv[2])
+int
+error_gen_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    error_gen_utimens_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->utimens,
-		    loc,
-		    tv);
-	return 0;
+	STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_access_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
-}
 
-int32_t
-error_gen_access (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  int32_t mask)
+int
+error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		  int32_t mask, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_ACCESS];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_ACCESS);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (access, frame, -1, op_errno, xdata);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_access_cbk,
+
+	STACK_WIND (frame, error_gen_access_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->access,
-		    loc,
-		    mask);
-	return 0;
+		    loc, mask, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_readlink_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			const char *path)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      path);
-	return 0;
+int
+error_gen_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			int32_t op_ret, int32_t op_errno,
+			const char *path, struct iatt *sbuf, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, sbuf, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_readlink (call_frame_t *frame,
-		    xlator_t *this,
-		    loc_t *loc,
-		    size_t size)
+
+int
+error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    size_t size, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_READLINK];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_READLINK);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_readlink_cbk,
+
+	STACK_WIND (frame, error_gen_readlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readlink,
-		    loc,
-		    size);
-	return 0;
+		    loc, size, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_mknod_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      inode,
-		      buf);
-	return 0;
+int
+error_gen_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno,
+                             inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_mknod (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 mode_t mode,
-		 dev_t rdev)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
+
+int
+error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		 mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_MKNOD];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_MKNOD);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
+                                     NULL, NULL, xdata);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_mknod_cbk,
+
+	STACK_WIND (frame, error_gen_mknod_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev);
-	return 0;
+		    loc, mode, rdev, umask, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_mkdir_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      inode,
-		      buf);
-	return 0;
+
+int
+error_gen_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno,
+                             inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_mkdir (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 mode_t mode)
+int
+error_gen_mkdir (call_frame_t *frame, xlator_t *this,
+		 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_MKDIR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_MKDIR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL,
+                                     NULL, NULL, xdata);
+                return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_mkdir_cbk,
+
+	STACK_WIND (frame, error_gen_mkdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode);
-	return 0;
+		    loc, mode, umask, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_unlink_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
+
+int
+error_gen_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		      int32_t op_ret, int32_t op_errno,
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_unlink (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc)
+
+int
+error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                  dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_UNLINK];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_UNLINK);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL,
+                                     xdata);
+                return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_unlink_cbk,
+
+	STACK_WIND (frame, error_gen_unlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->unlink,
-		    loc);
-	return 0;
+		    loc, xflag, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_rmdir_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
+
+int
+error_gen_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_rmdir (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc)
+
+int
+error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                 dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_RMDIR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_RMDIR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_rmdir_cbk,
+
+	STACK_WIND (frame, error_gen_rmdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->rmdir,
-		    loc);
-	return 0;
+		    loc, flags, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_symlink_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct stat *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode,	buf);
-	return 0;
+int
+error_gen_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_symlink (call_frame_t *frame,
-		   xlator_t *this,
-		   const char *linkpath,
-		   loc_t *loc)
+
+int
+error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+		   loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_SYMLINK];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_SYMLINK);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+		STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL,
+                                     NULL, NULL, NULL); /* pre & post parent attr */
 		return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_symlink_cbk,
+
+	STACK_WIND (frame, error_gen_symlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc);
-	return 0;
+		    linkpath, loc, umask, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_rename_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct stat *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
+int
+error_gen_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		      int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                      struct iatt *preoldparent, struct iatt *postoldparent,
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
+                             preoldparent, postoldparent,
+                             prenewparent, postnewparent, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_rename (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *oldloc,
-		  loc_t *newloc)
+
+int
+error_gen_rename (call_frame_t *frame, xlator_t *this,
+		  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_RENAME];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_RENAME);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
+		STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
+                                     NULL, NULL, NULL, NULL, NULL);
 		return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_rename_cbk,
+
+	STACK_WIND (frame, error_gen_rename_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->rename,
-		    oldloc, newloc);
-	return 0;
+		    oldloc, newloc, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_link_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-		    struct stat *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode,	buf);
-	return 0;
+int
+error_gen_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_link (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *oldloc,
-		loc_t *newloc)
+
+int
+error_gen_link (call_frame_t *frame, xlator_t *this,
+		loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int             op_errno = 0;
+        eg_t            *egp = NULL;
+        int             enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_LINK];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_LINK);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+		STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL,
+                                     NULL, NULL, NULL);
 		return 0;
 	}
-	STACK_WIND (frame,
-		    error_gen_link_cbk,
+
+	STACK_WIND (frame, error_gen_link_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->link,
-		    oldloc, newloc);
-	return 0;
+		    oldloc, newloc, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_create_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd,
-		      inode_t *inode,
-		      struct stat *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
-	return 0;
+int
+error_gen_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		      int32_t op_ret, int32_t op_errno,
+		      fd_t *fd, inode_t *inode, struct iatt *buf,
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_create (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  int32_t flags,
-		  mode_t mode, fd_t *fd)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
+
+int
+error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		  int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                  dict_t *xdata)
+{
+	int             op_errno = 0;
+        eg_t            *egp = NULL;
+        int             enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_CREATE];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_CREATE);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL);
+		STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL,
+                                     NULL, NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_create_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, fd);
-	return 0;
+		    loc, flags, mode, umask, fd, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_open_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    fd_t *fd)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      fd);
-	return 0;
+
+int
+error_gen_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_open (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		int32_t flags, fd_t *fd)
+
+int
+error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		int32_t flags, fd_t *fd, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_OPEN];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_OPEN);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_open_cbk,
+	STACK_WIND (frame, error_gen_open_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd);
-	return 0;
+		    loc, flags, fd, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_readv_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iovec *vector,
-		     int32_t count,
-		     struct stat *stbuf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      vector,
-		      count,
-		      stbuf);
-	return 0;
+
+int
+error_gen_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno,
+		     struct iovec *vector, int32_t count,
+		     struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
+                             vector, count, stbuf, iobref, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_readv (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 size_t size,
-		 off_t offset)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
+
+int
+error_gen_readv (call_frame_t *frame, xlator_t *this,
+		 fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_READ];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_READ);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0,
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 
-	STACK_WIND (frame,
-		    error_gen_readv_cbk,
+	STACK_WIND (frame, error_gen_readv_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readv,
-		    fd,
-		    size,
-		    offset);
-	return 0;
+		    fd, size, offset, flags, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_writev_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct stat *stbuf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      stbuf);
-	return 0;
+int
+error_gen_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		      int32_t op_ret, int32_t op_errno,
+                      struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_writev (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  struct iovec *vector,
-		  int32_t count,
-		  off_t off)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
+
+int
+error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		  struct iovec *vector, int32_t count,
+		  off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_WRITE];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_WRITE);
+
+	if (op_errno == GF_ERROR_SHORT_WRITE) {
+		struct iovec *shortvec;
+
+		/*
+		 * A short write error returns some value less than what was
+		 * requested from a write. To simulate this, replace the vector
+		 * with one half the size;
+		 */
+		shortvec = iov_dup(vector, 1);
+		shortvec->iov_len /= 2;
+
+		STACK_WIND(frame, error_gen_writev_cbk, FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev, fd, shortvec, count,
+			   off, flags, iobref, xdata);
+		GF_FREE(shortvec);
 		return 0;
+	} else if (op_errno) {
+		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
+		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, xdata);
+        	return 0;
 	}
 
-
-	STACK_WIND (frame,
-		    error_gen_writev_cbk,
+	STACK_WIND (frame, error_gen_writev_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->writev,
-		    fd,
-		    vector,
-		    count,
-		    off);
-	return 0;
+		    fd, vector, count, off, flags, iobref, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_flush_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
+
+int
+error_gen_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_flush (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd)
+
+int
+error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FLUSH];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FLUSH);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (flush, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_flush_cbk,
-		    FIRST_CHILD(this),
+	STACK_WIND (frame, error_gen_flush_cbk,
+                    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->flush,
-		    fd);
-	return 0;
+		    fd, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_fsync_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
+int
+error_gen_fsync_cbk (call_frame_t *frame, void *cookie,
+		     xlator_t *this, int32_t op_ret,
+		     int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_fsync (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 int32_t flags)
+
+int
+error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FSYNC];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FSYNC);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_fsync_cbk,
+	STACK_WIND (frame, error_gen_fsync_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsync,
-		    fd,
-		    flags);
-	return 0;
+		    fd, flags, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_fstat_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct stat *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
+
+int
+error_gen_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_fstat (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd)
+
+int
+error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FSTAT];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FSTAT);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_fstat_cbk,
+	STACK_WIND (frame, error_gen_fstat_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fstat,
-		    fd);
-	return 0;
+		    fd, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_opendir_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       fd_t *fd)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      fd);
-	return 0;
+
+int
+error_gen_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_opendir (call_frame_t *frame,
-		   xlator_t *this,
-		   loc_t *loc, fd_t *fd)
+
+int
+error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_OPENDIR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_OPENDIR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_opendir_cbk,
+	STACK_WIND (frame, error_gen_opendir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd);
-	return 0;
+		    loc, fd, xdata);
+        return 0;
 }
 
-
-int32_t
-error_gen_getdents_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			dir_entry_t *entries,
-			int32_t count)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      entries,
-		      count);
-	return 0;
+int
+error_gen_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_getdents (call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    size_t size,
-		    off_t offset,
-		    int32_t flag)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
+
+int
+error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    int32_t flags, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FSYNCDIR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FSYNCDIR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, 0);
-		return 0;
+		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_getdents_cbk,
+	STACK_WIND (frame, error_gen_fsyncdir_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getdents,
-		    fd,
-		    size,
-		    offset,
-		    flag);
-	return 0;
+		    FIRST_CHILD(this)->fops->fsyncdir,
+		    fd, flags, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_setdents_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
+int
+error_gen_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		      int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_setdents (call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    int32_t flags,
-		    dir_entry_t *entries,
-		    int32_t count)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
+
+int
+error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_STATFS];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_STATFS);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, 0);
-		return 0;
+		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_setdents_cbk,
+	STACK_WIND (frame, error_gen_statfs_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setdents,
-		    fd,
-		    flags,
-		    entries,
-		    count);
-	return 0;
+		    FIRST_CHILD(this)->fops->statfs,
+		    loc, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_fsyncdir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
+int
+error_gen_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_fsyncdir (call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    int32_t flags)
+
+int
+error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		    dict_t *dict, int32_t flags, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_SETXATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_SETXATTR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_fsyncdir_cbk,
+	STACK_WIND (frame, error_gen_setxattr_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsyncdir,
-		    fd,
-		    flags);
-	return 0;
+		    FIRST_CHILD(this)->fops->setxattr,
+		    loc, dict, flags, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_statfs_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct statvfs *buf)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
+int
+error_gen_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_statfs (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc)
+
+int
+error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		    const char *name, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_GETXATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_GETXATTR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_statfs_cbk,
+	STACK_WIND (frame, error_gen_getxattr_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->statfs,
-		    loc);
-	return 0;
+		    FIRST_CHILD(this)->fops->getxattr,
+		    loc, name, xdata);
+        return 0;
 }
 
-
-int32_t
-error_gen_setxattr_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
+int
+error_gen_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_setxattr (call_frame_t *frame,
-		    xlator_t *this,
-		    loc_t *loc,
-		    dict_t *dict,
-		    int32_t flags)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
+
+int
+error_gen_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     dict_t *dict, int32_t flags, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FSETXATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FSETXATTR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_setxattr_cbk,
+	STACK_WIND (frame, error_gen_fsetxattr_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setxattr,
-		    loc,
-		    dict,
-		    flags);
-	return 0;
+		    FIRST_CHILD(this)->fops->fsetxattr,
+		    fd, dict, flags, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_getxattr_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			dict_t *dict)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      dict);
-	return 0;
+
+int
+error_gen_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_getxattr (call_frame_t *frame,
-		    xlator_t *this,
-		    loc_t *loc,
-		    const char *name)
+
+int
+error_gen_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FGETXATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FGETXATTR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_getxattr_cbk,
+	STACK_WIND (frame, error_gen_fgetxattr_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc,
-		    name);
-	return 0;
+		    FIRST_CHILD(this)->fops->fgetxattr,
+		    fd, name, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_xattrop_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
+
+int
+error_gen_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_xattrop (call_frame_t *frame,
-		   xlator_t *this,
-		   loc_t *loc,
-		   gf_xattrop_flags_t flags,
-		   dict_t *dict)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
+
+int
+error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_XATTROP];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_XATTROP);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_xattrop_cbk,
+	STACK_WIND (frame, error_gen_xattrop_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->xattrop,
-		    loc, flags, dict);
-	return 0;
+		    loc, flags, dict, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_fxattrop_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
+
+int
+error_gen_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_fxattrop (call_frame_t *frame,
-  		    xlator_t *this,
-  		    fd_t *fd,
-  		    gf_xattrop_flags_t flags,
-  		    dict_t *dict)
-{
-  	int op_errno = 0;
-  	op_errno = error_gen(this);
-  	if (op_errno) {
-  		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-  		STACK_UNWIND (frame, -1, op_errno, NULL);
-  		return 0;
-  	}
-
-  	STACK_WIND (frame,
-  		    error_gen_fxattrop_cbk,
-  		    FIRST_CHILD(this),
-  		    FIRST_CHILD(this)->fops->fxattrop,
-  		    fd, flags, dict);
-  	return 0;
+
+int
+error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		    gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FXATTROP];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FXATTROP);
+
+	if (op_errno) {
+		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
+		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL, xdata);
+        return 0;
+	}
+
+	STACK_WIND (frame, error_gen_fxattrop_cbk,
+		    FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fxattrop,
+		    fd, flags, dict, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_removexattr_cbk (call_frame_t *frame,
-			   void *cookie,
-			   xlator_t *this,
-			   int32_t op_ret,
-			   int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
+
+int
+error_gen_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_removexattr (call_frame_t *frame,
-		       xlator_t *this,
-		       loc_t *loc,
-		       const char *name)
+
+int
+error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+		       const char *name, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_REMOVEXATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_REMOVEXATTR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_removexattr_cbk,
+	STACK_WIND (frame, error_gen_removexattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->removexattr,
-		    loc,
-		    name);
-	return 0;
+		    loc, name, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_lk_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct flock *lock)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      lock);
-	return 0;
+int
+error_gen_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_lk (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      int32_t cmd,
-	      struct flock *lock)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
+
+int
+error_gen_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        const char *name, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FREMOVEXATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FREMOVEXATTR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_lk_cbk,
+	STACK_WIND (frame, error_gen_fremovexattr_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lk,
-		    fd,
-		    cmd,
-		    lock);
-	return 0;
+		    FIRST_CHILD(this)->fops->fremovexattr,
+		    fd, name, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_inodelk_cbk (call_frame_t *frame, void *cookie,
-		       xlator_t *this, int32_t op_ret, int32_t op_errno)
+int
+error_gen_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
+        return 0;
+}
+
 
+int
+error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+	      struct gf_flock *lock, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_LK];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_LK);
+
+	if (op_errno) {
+		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
+		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL, xdata);
+        return 0;
+	}
+
+	STACK_WIND (frame, error_gen_lk_cbk,
+		    FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->lk,
+		    fd, cmd, lock, xdata);
+        return 0;
 }
 
 
-int32_t
+int
+error_gen_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
 error_gen_inodelk (call_frame_t *frame, xlator_t *this,
-		   loc_t *loc, int32_t cmd, struct flock *lock)
+		   const char *volume, loc_t *loc, int32_t cmd,
+                   struct gf_flock *lock, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_INODELK];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_INODELK);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_inodelk_cbk,
+	STACK_WIND (frame, error_gen_inodelk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->inodelk,
-		    loc, cmd, lock);
-	return 0;
+		    volume, loc, cmd, lock, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_finodelk_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+int
+error_gen_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
-int32_t
+int
 error_gen_finodelk (call_frame_t *frame, xlator_t *this,
-		    fd_t *fd, int32_t cmd, struct flock *lock)
+		    const char *volume, fd_t *fd, int32_t cmd,
+                    struct gf_flock *lock, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FINODELK];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FINODELK);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_finodelk_cbk,
+	STACK_WIND (frame, error_gen_finodelk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->finodelk,
-		    fd, cmd, lock);
-	return 0;
+		    volume, fd, cmd, lock, xdata);
+        return 0;
 }
 
 
-int32_t
-error_gen_entrylk_cbk (call_frame_t *frame, void *cookie,
-		       xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+int
+error_gen_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
+
+int
 error_gen_entrylk (call_frame_t *frame, xlator_t *this,
-		   loc_t *loc, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type)
+		   const char *volume, loc_t *loc, const char *basename,
+		   entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_ENTRYLK];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_ENTRYLK);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_entrylk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->entrylk,
-		    loc, basename, cmd, type);
-	return 0;
+		    volume, loc, basename, cmd, type, xdata);
+        return 0;
 }
 
-int32_t
-error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this, int32_t op_ret, int32_t op_errno)
 
+int
+error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t
+
+int
 error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
-		    fd_t *fd, const char *basename,
-		    entrylk_cmd cmd, entrylk_type type)
+		    const char *volume, fd_t *fd, const char *basename,
+		    entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int             op_errno = 0;
+        eg_t            *egp = NULL;
+        int             enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FENTRYLK];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FENTRYLK);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno, xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fentrylk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fentrylk,
-		    fd, basename, cmd, type);
-	return 0;
+		    volume, fd, basename, cmd, type, xdata);
+        return 0;
 }
 
 
 /* Management operations */
 
-int32_t
-error_gen_stats_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct xlator_stats *stats)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      stats);
-	return 0;
+
+int
+error_gen_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, char *spec_data)
+{
+	STACK_UNWIND_STRICT (getspec, frame, op_ret, op_errno, spec_data);
+        return 0;
 }
 
 
-int32_t
-error_gen_stats (call_frame_t *frame,
-		 xlator_t *this,
-		 int32_t flags)
+int
+error_gen_getspec (call_frame_t *frame, xlator_t *this, const char *key,
+		   int32_t flags)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_GETSPEC];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_GETSPEC);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
+		STACK_UNWIND_STRICT (getspec, frame, -1, op_errno, NULL);
 		return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_stats_cbk,
+	STACK_WIND (frame, error_gen_getspec_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->mops->stats,
-		    flags);
+		    FIRST_CHILD(this)->fops->getspec,
+		    key, flags);
 	return 0;
 }
 
 
-
-int32_t
-error_gen_getspec_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       char *spec_data)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      spec_data);
+int
+error_gen_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
 	return 0;
 }
 
 
-int32_t
-error_gen_getspec (call_frame_t *frame,
-		   xlator_t *this,
-		   const char *key,
-		   int32_t flags)
+int
+error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		   size_t size, off_t off, dict_t *xdata)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_READDIR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_READDIR);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
+		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL, xdata);
 		return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_getspec_cbk,
+	STACK_WIND (frame, error_gen_readdir_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->mops->getspec,
-		    key, flags);
+		    FIRST_CHILD(this)->fops->readdir,
+		    fd, size, off, xdata);
 	return 0;
 }
 
 
-int32_t
-error_gen_checksum_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			uint8_t *file_checksum,
-			uint8_t *dir_checksum)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      file_checksum,
-		      dir_checksum);
+int
+error_gen_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 	return 0;
 }
 
 
-int32_t
-error_gen_checksum (call_frame_t *frame,
-		    xlator_t *this,
-		    loc_t *loc,
-		    int32_t flag)
+int
+error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                    off_t off, dict_t *dict)
 {
-	int op_errno = 0;
-	op_errno = error_gen(this);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_READDIRP];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_READDIRP);
+
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
+		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
 		return 0;
 	}
 
-	STACK_WIND (frame,
-		    error_gen_checksum_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->checksum,
-		    loc,
-		    flag);
+	STACK_WIND (frame, error_gen_readdirp_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdirp,
+                    fd, size, off, dict);
 	return 0;
 }
 
-int32_t
-error_gen_readdir_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       gf_dirent_t *entries)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entries);
-	return 0;
+static void
+error_gen_set_failure (eg_t *pvt, int percent)
+{
+        GF_ASSERT (pvt);
+
+        if (percent)
+                pvt->failure_iter_no = 100/percent;
+        else
+                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
 }
 
+static void
+error_gen_parse_fill_fops (eg_t *pvt, char *enable_fops)
+{
+        char            *op_no_str = NULL;
+        int              op_no = -1;
+        int              i = 0;
+        xlator_t        *this = THIS;
+        char            *saveptr = NULL;
 
-int32_t
-error_gen_readdir (call_frame_t *frame,
-		   xlator_t *this,
-		   fd_t *fd,
-		   size_t size,
-		   off_t off)
-{
-	int op_errno = 0;
-	op_errno = error_gen(this);
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND (frame, -1, op_errno, NULL);
-		return 0;
-	}
+        GF_ASSERT (pvt);
+        GF_ASSERT (this);
 
-	STACK_WIND (frame,
-		    error_gen_readdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readdir,
-		    fd, size, off);
-	return 0;
+        for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                pvt->enable[i] = 0;
+
+        if (!enable_fops) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "All fops are enabled.");
+                for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                        pvt->enable[i] = 1;
+        } else {
+                op_no_str = strtok_r (enable_fops, ",", &saveptr);
+                while (op_no_str) {
+                        op_no = get_fop_int (&op_no_str);
+                        if (op_no == -1) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Wrong option value %s", op_no_str);
+                        } else
+                                pvt->enable[op_no] = 1;
+
+                        op_no_str = strtok_r (NULL, ",", &saveptr);
+                }
+        }
 }
 
 int32_t
-error_gen_closedir (xlator_t *this,
-		    fd_t *fd)
+error_gen_priv_dump (xlator_t *this)
 {
-	return 0;
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        int             ret = -1;
+        eg_t            *conf = NULL;
+
+        if (!this)
+            goto out;
+
+        conf = this->private;
+        if (!conf)
+            goto out;
+
+        ret = TRY_LOCK(&conf->lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.debug.error-gen","%s.priv",
+                               this->name);
+
+        gf_proc_dump_write("op_count", "%d", conf->op_count);
+        gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
+        gf_proc_dump_write("error_no", "%s", conf->error_no);
+        gf_proc_dump_write("random_failure", "%d", conf->random_failure);
+
+        UNLOCK(&conf->lock);
+out:
+        return ret;
 }
 
 int32_t
-error_gen_close (xlator_t *this,
-		 fd_t *fd)
+mem_acct_init (xlator_t *this)
 {
-	return 0;
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_error_gen_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        eg_t            *pvt = NULL;
+        int32_t          ret = 0;
+        char            *error_enable_fops = NULL;
+        int32_t          failure_percent_int = 0;
+
+        if (!this || !this->private)
+                goto out;
+
+        pvt = this->private;
+
+        GF_OPTION_RECONF ("error-no", pvt->error_no, options, str, out);
+
+        GF_OPTION_RECONF ("failure", failure_percent_int, options, int32,
+                          out);
+
+        GF_OPTION_RECONF ("enable", error_enable_fops, options, str, out);
+
+        GF_OPTION_RECONF ("random-failure", pvt->random_failure, options,
+                          bool, out);
+
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
+        return ret;
 }
 
 int
 init (xlator_t *this)
 {
-	eg_t *pvt = NULL;
+        eg_t            *pvt = NULL;
+        int32_t          ret = 0;
+        char            *error_enable_fops = NULL;
+        int32_t          failure_percent_int = 0;
 
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"error-gen not configured with one subvolume");
-		return -1;
-	}
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error-gen not configured with one subvolume");
+                ret = -1;
+                goto out;
+        }
 
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	pvt = CALLOC (1, sizeof (eg_t));
-	this->private = pvt;
-	return 0;
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+        pvt = GF_CALLOC (1, sizeof (eg_t), gf_error_gen_mt_eg_t);
+
+        if (!pvt) {
+                ret = -1;
+                goto out;
+        }
+
+        LOCK_INIT (&pvt->lock);
+
+        GF_OPTION_INIT ("error-no", pvt->error_no, str, out);
+
+        GF_OPTION_INIT ("failure", failure_percent_int, int32, out);
+
+        GF_OPTION_INIT ("enable", error_enable_fops, str, out);
+
+        GF_OPTION_INIT ("random-failure", pvt->random_failure, bool, out);
+
+
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
+
+        this->private = pvt;
+
+        /* Give some seed value here */
+        srand (time(NULL));
+out:
+        if (ret)
+                GF_FREE (pvt);
+        return ret;
 }
 
+
 void
 fini (xlator_t *this)
 {
-	gf_log (this->name, GF_LOG_DEBUG, "fini called");
-	return;
+        eg_t            *pvt = NULL;
+
+        if (!this)
+                 return;
+        pvt = this->private;
+
+        if (pvt) {
+                LOCK_DESTROY (&pvt->lock);
+                GF_FREE (pvt);
+                gf_log (this->name, GF_LOG_DEBUG, "fini called");
+        }
+        return;
 }
 
+struct xlator_dumpops dumpops = {
+        .priv = error_gen_priv_dump,
+};
+
+struct xlator_fops cbks;
 
 struct xlator_fops fops = {
 	.lookup      = error_gen_lookup,
@@ -1734,10 +2170,7 @@ struct xlator_fops fops = {
 	.symlink     = error_gen_symlink,
 	.rename      = error_gen_rename,
 	.link        = error_gen_link,
-	.chmod       = error_gen_chmod,
-	.chown       = error_gen_chown,
 	.truncate    = error_gen_truncate,
-	.utimens     = error_gen_utimens,
 	.create      = error_gen_create,
 	.open        = error_gen_open,
 	.readv       = error_gen_readv,
@@ -1748,33 +2181,52 @@ struct xlator_fops fops = {
 	.setxattr    = error_gen_setxattr,
 	.getxattr    = error_gen_getxattr,
 	.removexattr = error_gen_removexattr,
+	.fsetxattr    = error_gen_fsetxattr,
+	.fgetxattr    = error_gen_fgetxattr,
+	.fremovexattr = error_gen_fremovexattr,
 	.opendir     = error_gen_opendir,
 	.readdir     = error_gen_readdir,
-	.getdents    = error_gen_getdents,
+	.readdirp    = error_gen_readdirp,
 	.fsyncdir    = error_gen_fsyncdir,
 	.access      = error_gen_access,
 	.ftruncate   = error_gen_ftruncate,
 	.fstat       = error_gen_fstat,
 	.lk          = error_gen_lk,
-	.fchmod      = error_gen_fchmod,
-	.fchown      = error_gen_fchown,
-	.setdents    = error_gen_setdents,
 	.lookup_cbk  = error_gen_lookup_cbk,
-	.checksum    = error_gen_checksum,
 	.xattrop     = error_gen_xattrop,
 	.fxattrop    = error_gen_fxattrop,
 	.inodelk     = error_gen_inodelk,
 	.finodelk    = error_gen_finodelk,
 	.entrylk     = error_gen_entrylk,
-	.fentrylk    = error_gen_fentrylk
-};
-
-struct xlator_mops mops = {
-	.stats = error_gen_stats,
-	.getspec = error_gen_getspec,
+	.fentrylk    = error_gen_fentrylk,
+        .setattr     = error_gen_setattr,
+        .fsetattr    = error_gen_fsetattr,
+	.getspec     = error_gen_getspec,
 };
 
-struct xlator_cbks cbks = {
-	.release = error_gen_close,
-	.releasedir = error_gen_closedir,
+struct volume_options options[] = {
+        { .key  = {"failure"},
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Percentage failure of operations when enabled.",
+        },
+
+        { .key  = {"error-no"},
+          .value = {"ENOENT","ENOTDIR","ENAMETOOLONG","EACCES","EBADF",
+                    "EFAULT","ENOMEM","EINVAL","EIO","EEXIST","ENOSPC",
+                    "EPERM","EROFS","EBUSY","EISDIR","ENOTEMPTY","EMLINK"
+                    "ENODEV","EXDEV","EMFILE","ENFILE","ENOSYS","EINTR",
+                    "EFBIG","EAGAIN","GF_ERROR_SHORT_WRITE"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
+        { .key = {"random-failure"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+
+        { .key  = {"enable"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
+        { .key  = {NULL} }
 };
diff --git a/xlators/debug/error-gen/src/error-gen.h b/xlators/debug/error-gen/src/error-gen.h
new file mode 100644
index 000000000..d92c23062
--- /dev/null
+++ b/xlators/debug/error-gen/src/error-gen.h
@@ -0,0 +1,48 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _ERROR_GEN_H
+#define _ERROR_GEN_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "error-gen-mem-types.h"
+
+#define GF_FAILURE_DEFAULT 10
+
+/*
+ * Pseudo-errors refer to errors beyond the scope of traditional <-1, op_errno>
+ * returns. This facilitates the ability to return unexpected, but not -1 values
+ * and/or to inject operations that lead to implicit error conditions. The range
+ * for pseudo errors resides at a high value to avoid conflicts with the errno
+ * range.
+ */
+enum GF_PSEUDO_ERRORS {
+	GF_ERROR_SHORT_WRITE = 1000,	/* short writev return value */
+	GF_ERROR_MAX
+};
+
+typedef struct {
+        int enable[GF_FOP_MAXVALUE];
+        int op_count;
+        int failure_iter_no;
+        char *error_no;
+        gf_boolean_t random_failure;
+        gf_lock_t lock;
+} eg_t;
+
+typedef struct {
+        int error_no_count;
+	int error_no[20];
+} sys_error_t;
+
+#endif
diff --git a/xlators/debug/io-stats/Makefile.am b/xlators/debug/io-stats/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/debug/io-stats/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/debug/io-stats/src/Makefile.am b/xlators/debug/io-stats/src/Makefile.am
new file mode 100644
index 000000000..332d79015
--- /dev/null
+++ b/xlators/debug/io-stats/src/Makefile.am
@@ -0,0 +1,16 @@
+
+xlator_LTLIBRARIES = io-stats.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
+
+io_stats_la_LDFLAGS = -module -avoid-version
+
+io_stats_la_SOURCES = io-stats.c
+io_stats_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = io-stats-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
new file mode 100644
index 000000000..c30dfb17e
--- /dev/null
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -0,0 +1,24 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __IO_STATS_MEM_TYPES_H__
+#define __IO_STATS_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_io_stats_mem_types_ {
+        gf_io_stats_mt_ios_conf = gf_common_mt_end + 1,
+        gf_io_stats_mt_ios_fd,
+        gf_io_stats_mt_ios_stat,
+        gf_io_stats_mt_ios_stat_list,
+        gf_io_stats_mt_end
+};
+#endif
+
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
new file mode 100644
index 000000000..7fb697ae4
--- /dev/null
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -0,0 +1,2950 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#include "xlator.h"
+#endif
+
+/**
+ * xlators/debug/io_stats :
+ *    This translator maintains statistics of all filesystem activity
+ *    happening through it. The kind of statistics include:
+ *
+ *  a) total read data - since process start, last interval and per fd
+ *  b) total write data - since process start, last interval and per fd
+ *  c) counts of read IO block size - since process start, last interval and per fd
+ *  d) counts of write IO block size - since process start, last interval and per fd
+ *  e) counts of all FOP types passing through it
+ *
+ *  Usage: setfattr -n io-stats-dump /tmp/filename /mnt/gluster
+ *
+ */
+
+#include <fnmatch.h>
+#include <errno.h>
+#include "glusterfs.h"
+#include "xlator.h"
+#include "io-stats-mem-types.h"
+#include <stdarg.h>
+#include "defaults.h"
+#include "logging.h"
+
+#define MAX_LIST_MEMBERS 100
+
+typedef enum {
+        IOS_STATS_TYPE_NONE,
+        IOS_STATS_TYPE_OPEN,
+        IOS_STATS_TYPE_READ,
+        IOS_STATS_TYPE_WRITE,
+        IOS_STATS_TYPE_OPENDIR,
+        IOS_STATS_TYPE_READDIRP,
+        IOS_STATS_TYPE_READ_THROUGHPUT,
+        IOS_STATS_TYPE_WRITE_THROUGHPUT,
+        IOS_STATS_TYPE_MAX
+}ios_stats_type_t;
+
+typedef enum {
+        IOS_STATS_THRU_READ,
+        IOS_STATS_THRU_WRITE,
+        IOS_STATS_THRU_MAX,
+}ios_stats_thru_t;
+
+struct ios_stat_lat {
+        struct timeval  time;
+        double          throughput;
+};
+
+struct ios_stat {
+        gf_lock_t       lock;
+        uuid_t          gfid;
+        char           *filename;
+        uint64_t        counters [IOS_STATS_TYPE_MAX];
+        struct ios_stat_lat thru_counters [IOS_STATS_THRU_MAX];
+        int             refcnt;
+};
+
+struct ios_stat_list {
+        struct list_head  list;
+        struct ios_stat  *iosstat;
+        double            value;
+};
+
+struct ios_stat_head {
+       gf_lock_t                lock;
+       double                   min_cnt;
+       uint64_t                 members;
+       struct ios_stat_list    *iosstats;
+};
+
+struct ios_lat {
+        double  min;
+        double  max;
+        double  avg;
+};
+
+struct ios_global_stats {
+        uint64_t        data_written;
+        uint64_t        data_read;
+        uint64_t        block_count_write[32];
+        uint64_t        block_count_read[32];
+        uint64_t        fop_hits[GF_FOP_MAXVALUE];
+        struct timeval  started_at;
+        struct ios_lat  latency[GF_FOP_MAXVALUE];
+        uint64_t        nr_opens;
+        uint64_t        max_nr_opens;
+        struct timeval  max_openfd_time;
+};
+
+
+struct ios_conf {
+        gf_lock_t                 lock;
+        struct ios_global_stats   cumulative;
+        uint64_t                  increment;
+        struct ios_global_stats   incremental;
+        gf_boolean_t              dump_fd_stats;
+        gf_boolean_t              count_fop_hits;
+        gf_boolean_t              measure_latency;
+        struct ios_stat_head      list[IOS_STATS_TYPE_MAX];
+        struct ios_stat_head      thru_list[IOS_STATS_THRU_MAX];
+};
+
+
+struct ios_fd {
+        char           *filename;
+        uint64_t        data_written;
+        uint64_t        data_read;
+        uint64_t        block_count_write[32];
+        uint64_t        block_count_read[32];
+        struct timeval  opened_at;
+};
+
+typedef enum {
+        IOS_DUMP_TYPE_NONE = 0,
+        IOS_DUMP_TYPE_FILE = 1,
+        IOS_DUMP_TYPE_DICT = 2,
+        IOS_DUMP_TYPE_MAX  = 3
+} ios_dump_type_t;
+
+struct ios_dump_args {
+        ios_dump_type_t type;
+        union {
+                FILE *logfp;
+                dict_t *dict;
+        } u;
+};
+
+typedef int (*block_dump_func) (xlator_t *, struct ios_dump_args*,
+                                    int , int , uint64_t ) ;
+
+struct ios_local {
+        struct timeval  wind_at;
+        struct timeval  unwind_at;
+};
+
+struct volume_options options[];
+
+inline static int
+is_fop_latency_started (call_frame_t *frame)
+{
+        GF_ASSERT (frame);
+        struct timeval epoch = {0,};
+        return memcmp (&frame->begin, &epoch, sizeof (epoch));
+}
+
+#define END_FOP_LATENCY(frame, op)                                      \
+        do {                                                            \
+                struct ios_conf  *conf = NULL;                          \
+                                                                        \
+                conf = this->private;                                   \
+                if (conf && conf->measure_latency) {                    \
+                        gettimeofday (&frame->end, NULL);               \
+                        update_ios_latency (conf, frame, GF_FOP_##op);  \
+                }                                                       \
+        } while (0)
+
+#define START_FOP_LATENCY(frame)                                         \
+        do {                                                             \
+                struct ios_conf  *conf = NULL;                           \
+                                                                         \
+                conf = this->private;                                    \
+                if (conf && conf->measure_latency) {                     \
+                        gettimeofday (&frame->begin, NULL);              \
+                } else {                                                 \
+                        memset (&frame->begin, 0, sizeof (frame->begin));\
+                }                                                        \
+        } while (0)
+
+
+#define BUMP_FOP(op)                                                    \
+        do {                                                            \
+                struct ios_conf  *conf = NULL;                          \
+                                                                        \
+                conf = this->private;                                   \
+                if (!conf)                                              \
+                        break;                                          \
+                conf->cumulative.fop_hits[GF_FOP_##op]++;               \
+                conf->incremental.fop_hits[GF_FOP_##op]++;              \
+        } while (0)
+
+#define UPDATE_PROFILE_STATS(frame, op)                                       \
+        do {                                                                  \
+                struct ios_conf  *conf = NULL;                                \
+                                                                              \
+                if (!is_fop_latency_started (frame))                          \
+                        break;                                                \
+                conf = this->private;                                         \
+                LOCK (&conf->lock);                                           \
+                {                                                             \
+                        if (conf && conf->measure_latency &&                  \
+                            conf->count_fop_hits) {                           \
+                                BUMP_FOP(op);                                 \
+                                gettimeofday (&frame->end, NULL);             \
+                                update_ios_latency (conf, frame, GF_FOP_##op);\
+                        }                                                     \
+                }                                                             \
+                UNLOCK (&conf->lock);                                         \
+        } while (0)
+
+#define BUMP_READ(fd, len)                                              \
+        do {                                                            \
+                struct ios_conf  *conf = NULL;                          \
+                struct ios_fd    *iosfd = NULL;                         \
+                int               lb2 = 0;                              \
+                                                                        \
+                conf = this->private;                                   \
+                lb2 = log_base2 (len);                                  \
+                ios_fd_ctx_get (fd, this, &iosfd);                      \
+                if (!conf)                                              \
+                        break;                                          \
+                                                                        \
+                LOCK (&conf->lock);                                     \
+                {                                                       \
+                        conf->cumulative.data_read += len;              \
+                        conf->incremental.data_read += len;             \
+                        conf->cumulative.block_count_read[lb2]++;       \
+                        conf->incremental.block_count_read[lb2]++;      \
+                                                                        \
+                        if (iosfd) {                                    \
+                                iosfd->data_read += len;                \
+                                iosfd->block_count_read[lb2]++;         \
+                        }                                               \
+                }                                                       \
+                UNLOCK (&conf->lock);                                   \
+        } while (0)
+
+
+#define BUMP_WRITE(fd, len)                                             \
+        do {                                                            \
+                struct ios_conf  *conf = NULL;                          \
+                struct ios_fd    *iosfd = NULL;                         \
+                int               lb2 = 0;                              \
+                                                                        \
+                conf = this->private;                                   \
+                lb2 = log_base2 (len);                                  \
+                ios_fd_ctx_get (fd, this, &iosfd);                      \
+                if (!conf)                                              \
+                        break;                                          \
+                                                                        \
+                LOCK (&conf->lock);                                     \
+                {                                                       \
+                        conf->cumulative.data_written += len;           \
+                        conf->incremental.data_written += len;          \
+                        conf->cumulative.block_count_write[lb2]++;      \
+                        conf->incremental.block_count_write[lb2]++;     \
+                                                                        \
+                        if (iosfd) {                                    \
+                                iosfd->data_written += len;             \
+                                iosfd->block_count_write[lb2]++;        \
+                        }                                               \
+                }                                                       \
+                UNLOCK (&conf->lock);                                   \
+        } while (0)
+
+
+#define BUMP_STATS(iosstat, type)                                               \
+        do {                                                                    \
+                struct ios_conf         *conf = NULL;                           \
+                uint64_t                 value = 0;                             \
+                                                                                \
+                conf = this->private;                                           \
+                                                                                \
+                LOCK(&iosstat->lock);                                           \
+                {                                                               \
+                        iosstat->counters[type]++;                              \
+                        value = iosstat->counters[type];                        \
+                }                                                               \
+                UNLOCK (&iosstat->lock);                                        \
+                ios_stat_add_to_list (&conf->list[type],                        \
+                                     value, iosstat);                           \
+                                                                                \
+        } while (0)
+
+
+#define BUMP_THROUGHPUT(iosstat, type)						\
+        do {									\
+                struct ios_conf         *conf = NULL;				\
+                double                   elapsed;				\
+                struct timeval          *begin, *end;				\
+                double                   throughput;				\
+               int                      flag = 0;                              \
+                                                                                \
+                begin = &frame->begin;						\
+                end   = &frame->end;						\
+                                                                                \
+                elapsed = (end->tv_sec - begin->tv_sec) * 1e6			\
+                        + (end->tv_usec - begin->tv_usec);			\
+                throughput = op_ret / elapsed;					\
+                                                                                \
+                conf = this->private;						\
+                LOCK(&iosstat->lock);						\
+                {								\
+                        if (iosstat->thru_counters[type].throughput             \
+                                <= throughput) {                                \
+                                iosstat->thru_counters[type].throughput =       \
+                                                                throughput;     \
+                                gettimeofday (&iosstat->                        \
+                                             thru_counters[type].time, NULL);   \
+                               flag = 1;                                       \
+                        }							\
+                }								\
+                UNLOCK (&iosstat->lock);					\
+               if (flag)                                                       \
+                       ios_stat_add_to_list (&conf->thru_list[type],           \
+                                               throughput, iosstat);           \
+        } while (0)
+
+int
+ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
+{
+        uint64_t      iosfd64 = 0;
+        unsigned long iosfdlong = 0;
+        int           ret = 0;
+
+        ret = fd_ctx_get (fd, this, &iosfd64);
+        iosfdlong = iosfd64;
+        if (ret != -1)
+                *iosfd = (void *) iosfdlong;
+
+        return ret;
+}
+
+
+
+int
+ios_fd_ctx_set (fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
+{
+        uint64_t   iosfd64 = 0;
+        int        ret = 0;
+
+        iosfd64 = (unsigned long) iosfd;
+        ret = fd_ctx_set (fd, this, iosfd64);
+
+        return ret;
+}
+
+int
+ios_stat_ref (struct ios_stat *iosstat)
+{
+        LOCK (&iosstat->lock);
+        {
+                iosstat->refcnt++;
+        }
+        UNLOCK (&iosstat->lock);
+
+        return iosstat->refcnt;
+}
+
+int
+ios_stat_unref (struct ios_stat *iosstat)
+{
+        int cleanup = 0;
+        LOCK (&iosstat->lock);
+        {
+                iosstat->refcnt--;
+                if (iosstat->refcnt == 0) {
+                        if (iosstat->filename) {
+                                GF_FREE (iosstat->filename);
+                                iosstat->filename = NULL;
+                        }
+                        cleanup = 1;
+                }
+        }
+        UNLOCK (&iosstat->lock);
+
+        if (cleanup) {
+                GF_FREE (iosstat);
+                iosstat = NULL;
+        }
+
+        return 0;
+}
+
+int
+ios_inode_ctx_set (inode_t *inode, xlator_t *this, struct ios_stat *iosstat)
+{
+        uint64_t   iosstat64 = 0;
+        int        ret     = 0;
+
+        ios_stat_ref (iosstat);
+        iosstat64 = (unsigned long )iosstat;
+        ret = inode_ctx_put (inode, this, iosstat64);
+        return ret;
+}
+
+int
+ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
+{
+        uint64_t      iosstat64 = 0;
+        unsigned long iosstatlong = 0;
+        int           ret = 0;
+
+        ret = inode_ctx_get (inode, this, &iosstat64);
+        iosstatlong = iosstat64;
+        if (ret != -1)
+                *iosstat = (void *) iosstatlong;
+
+        return ret;
+
+}
+
+int
+ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
+                            struct ios_stat *iosstat)
+{
+        struct ios_stat_list *new = NULL;
+        struct ios_stat_list *entry = NULL;
+        struct ios_stat_list *t = NULL;
+        struct ios_stat_list *list_entry = NULL;
+        struct ios_stat_list *tmp = NULL;
+        struct ios_stat_list *last = NULL;
+        struct ios_stat      *stat = NULL;
+        int                   cnt = 0;
+        int                   found = 0;
+        int                   reposition = 0;
+        double                min_count = 0;
+
+        LOCK (&list_head->lock);
+        {
+
+                if (list_head->min_cnt == 0)
+                        list_head->min_cnt = value;
+                if ((list_head->members == MAX_LIST_MEMBERS) &&
+                     (list_head->min_cnt > value))
+                        goto out;
+
+                list_for_each_entry_safe (entry, t,
+                              &list_head->iosstats->list, list) {
+                        cnt++;
+                        if (cnt == list_head->members)
+                                last = entry;
+
+                        if (!uuid_compare (iosstat->gfid,
+                            entry->iosstat->gfid)) {
+                                list_entry = entry;
+                                found = cnt;
+                                entry->value = value;
+                                if (!reposition) {
+                                        if (cnt == list_head->members)
+                                                list_head->min_cnt = value;
+                                        goto out;
+                                }
+                                break;
+                        } else if (entry->value <= value && !reposition) {
+                                reposition = cnt;
+                                tmp = entry;
+                                if (cnt == list_head->members - 1)
+                                        min_count = entry->value;
+                        }
+                }
+                if (found) {
+                        list_del (&list_entry->list);
+                        list_add_tail (&list_entry->list, &tmp->list);
+                        if (min_count)
+                                list_head->min_cnt = min_count;
+                        goto out;
+                } else if (list_head->members == MAX_LIST_MEMBERS && reposition) {
+                        new = GF_CALLOC (1, sizeof (*new),
+                                        gf_io_stats_mt_ios_stat_list);
+                        new->iosstat = iosstat;
+                        new->value = value;
+                        ios_stat_ref (iosstat);
+                        list_add_tail (&new->list, &tmp->list);
+                        stat = last->iosstat;
+                        last->iosstat = NULL;
+                        ios_stat_unref (stat);
+                        list_del (&last->list);
+                        GF_FREE (last);
+                        if (reposition == MAX_LIST_MEMBERS)
+                                list_head->min_cnt = value;
+                        else if (min_count) {
+                                list_head->min_cnt = min_count;
+                        }
+                } else if (list_head->members < MAX_LIST_MEMBERS) {
+                        new  = GF_CALLOC (1, sizeof (*new),
+                                          gf_io_stats_mt_ios_stat_list);
+                        new->iosstat = iosstat;
+                        new->value = value;
+                        ios_stat_ref (iosstat);
+                        if (reposition) {
+                                list_add_tail (&new->list, &tmp->list);
+                        } else {
+                                list_add_tail (&new->list, &entry->list);
+                        }
+                        list_head->members++;
+                        if (list_head->min_cnt > value)
+                                list_head->min_cnt = value;
+                }
+        }
+out:
+        UNLOCK (&list_head->lock);
+        return 0;
+}
+
+inline int
+ios_stats_cleanup (xlator_t *this, inode_t *inode)
+{
+
+        struct ios_stat *iosstat = NULL;
+        uint64_t         iosstat64 = 0;
+
+        inode_ctx_del (inode, this, &iosstat64);
+        if (!iosstat64) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "could not get inode ctx");
+                return 0;
+        }
+        iosstat = (void *) (long)iosstat64;
+        if (iosstat) {
+                ios_stat_unref (iosstat);
+        }
+        return 0;
+}
+
+#define ios_log(this, logfp, fmt ...)                           \
+        do {                                                    \
+                if (logfp) {                                    \
+                        fprintf (logfp, fmt);                   \
+                        fprintf (logfp, "\n");                  \
+                }                                               \
+                gf_log (this->name, GF_LOG_INFO, fmt);        \
+        } while (0)
+
+int
+ios_dump_file_stats (struct ios_stat_head *list_head, xlator_t *this, FILE* logfp)
+{
+        struct ios_stat_list *entry = NULL;
+
+        LOCK (&list_head->lock);
+        {
+                list_for_each_entry (entry, &list_head->iosstats->list, list) {
+                        ios_log (this, logfp, "%-12.0f %s",
+                                entry->value, entry->iosstat->filename);
+                }
+        }
+        UNLOCK (&list_head->lock);
+        return 0;
+}
+
+int
+ios_dump_throughput_stats (struct ios_stat_head *list_head, xlator_t *this,
+                            FILE* logfp, ios_stats_type_t type)
+{
+        struct ios_stat_list *entry = NULL;
+        struct timeval        time  = {0, };
+        char                  timestr[256] = {0, };
+
+        LOCK (&list_head->lock);
+        {
+                list_for_each_entry (entry, &list_head->iosstats->list, list) {
+                        gf_time_fmt (timestr, sizeof timestr,
+                                     entry->iosstat->thru_counters[type].time.tv_sec,
+                                     gf_timefmt_FT);
+                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, time.tv_usec);
+
+                        ios_log (this, logfp, "%s \t %-10.2f  \t  %s",
+                                 timestr, entry->value, entry->iosstat->filename);
+                }
+        }
+        UNLOCK (&list_head->lock);
+        return 0;
+}
+
+int
+io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
+                               struct timeval *now, int interval, FILE* logfp)
+{
+        int                   i = 0;
+        int                   per_line = 0;
+        int                   index = 0;
+        struct ios_stat_head *list_head = NULL;
+        struct ios_conf      *conf = NULL;
+        char                  timestr[256] = {0, };
+        char                  str_header[128] = {0};
+        char                  str_read[128] = {0};
+        char                  str_write[128] = {0};
+
+        conf = this->private;
+
+        if (interval == -1)
+                ios_log (this, logfp, "\n=== Cumulative stats ===");
+        else
+                ios_log (this, logfp, "\n=== Interval %d stats ===",
+                         interval);
+        ios_log (this, logfp, "      Duration : %"PRId64" secs",
+                 (uint64_t) (now->tv_sec - stats->started_at.tv_sec));
+        ios_log (this, logfp, "     BytesRead : %"PRId64,
+                 stats->data_read);
+        ios_log (this, logfp, "  BytesWritten : %"PRId64"\n",
+                 stats->data_written);
+
+        snprintf (str_header, sizeof (str_header), "%-12s %c", "Block Size", ':');
+        snprintf (str_read, sizeof (str_read), "%-12s %c", "Read Count", ':');
+        snprintf (str_write, sizeof (str_write), "%-12s %c", "Write Count", ':');
+        index = 14;
+        for (i = 0; i < 32; i++) {
+                if ((stats->block_count_read[i] == 0) &&
+                    (stats->block_count_write[i] == 0))
+                        continue;
+                per_line++;
+
+                snprintf (str_header+index, sizeof (str_header)-index,
+                          "%16dB+", (1<<i));
+                if (stats->block_count_read[i])
+                        snprintf (str_read+index, sizeof (str_read)-index,
+                                  "%18"PRId64, stats->block_count_read[i]);
+                else    snprintf (str_read+index, sizeof (str_read)-index,
+                                  "%18s", "0");
+                if (stats->block_count_write[i])
+                        snprintf (str_write+index, sizeof (str_write)-index,
+                                  "%18"PRId64, stats->block_count_write[i]);
+                else    snprintf (str_write+index, sizeof (str_write)-index,
+                                  "%18s", "0");
+
+                index += 18;
+                if (per_line == 3) {
+                        ios_log (this, logfp, "%s", str_header);
+                        ios_log (this, logfp, "%s", str_read);
+                        ios_log (this, logfp, "%s\n", str_write);
+
+                        memset (str_header, 0, sizeof (str_header));
+                        memset (str_read, 0, sizeof (str_read));
+                        memset (str_write, 0, sizeof (str_write));
+
+                        snprintf (str_header, sizeof (str_header), "%-12s %c",
+                                  "Block Size", ':');
+                        snprintf (str_read, sizeof (str_read), "%-12s %c",
+                                  "Read Count", ':');
+                        snprintf (str_write, sizeof (str_write), "%-12s %c",
+                                  "Write Count", ':');
+
+                        index = 14;
+                        per_line = 0;
+                }
+        }
+
+        if (per_line != 0) {
+                ios_log (this, logfp, "%s", str_header);
+                ios_log (this, logfp, "%s", str_read);
+                ios_log (this, logfp, "%s\n", str_write);
+        }
+
+        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "Fop",
+                 "Call Count", "Avg-Latency", "Min-Latency",
+                 "Max-Latency");
+        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "---", "----------",
+                 "-----------", "-----------", "-----------");
+
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                if (stats->fop_hits[i] && !stats->latency[i].avg)
+                        ios_log (this, logfp, "%-13s %10"PRId64" %11s "
+                                 "us %11s us %11s us", gf_fop_list[i],
+                                 stats->fop_hits[i], "0", "0", "0");
+                else if (stats->fop_hits[i] && stats->latency[i].avg)
+                        ios_log (this, logfp, "%-13s %10"PRId64" %11.2lf us "
+                                 "%11.2lf us %11.2lf us", gf_fop_list[i],
+                                 stats->fop_hits[i], stats->latency[i].avg,
+                                 stats->latency[i].min, stats->latency[i].max);
+        }
+        ios_log (this, logfp, "------ ----- ----- ----- ----- ----- ----- ----- "
+                 " ----- ----- ----- -----\n");
+
+        if (interval == -1) {
+                LOCK (&conf->lock);
+                {
+                        gf_time_fmt (timestr, sizeof timestr,
+                                     conf->cumulative.max_openfd_time.tv_sec,
+                                     gf_timefmt_FT);
+                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                                  ".%"GF_PRI_SUSECONDS,
+                                  conf->cumulative.max_openfd_time.tv_usec);
+                        ios_log (this, logfp, "Current open fd's: %"PRId64
+                                 " Max open fd's: %"PRId64" time %s",
+                                 conf->cumulative.nr_opens,
+                                 conf->cumulative.max_nr_opens, timestr);
+                }
+                UNLOCK (&conf->lock);
+                ios_log (this, logfp, "\n==========Open File Stats========");
+                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+                ios_dump_file_stats (list_head, this, logfp);
+
+
+                ios_log (this, logfp, "\n==========Read File Stats========");
+                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_READ];
+                ios_dump_file_stats (list_head, this, logfp);
+
+                ios_log (this, logfp, "\n==========Write File Stats========");
+                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+                ios_dump_file_stats (list_head, this, logfp);
+
+                ios_log (this, logfp, "\n==========Directory open stats========");
+                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+                ios_dump_file_stats (list_head, this, logfp);
+
+                ios_log (this, logfp, "\n========Directory readdirp Stats=======");
+                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+                ios_dump_file_stats (list_head, this, logfp);
+
+                ios_log (this, logfp, "\n========Read Throughput File Stats=====");
+                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                         "\tFILE NAME");
+                list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+                ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
+
+                ios_log (this, logfp, "\n======Write Throughput File Stats======");
+                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                         "\tFILE NAME");
+                list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+                ios_dump_throughput_stats (list_head, this, logfp, IOS_STATS_THRU_WRITE);
+        }
+        return 0;
+}
+
+int
+io_stats_dump_global_to_dict (xlator_t *this, struct ios_global_stats *stats,
+                              struct timeval *now, int interval, dict_t *dict)
+{
+        int             ret = 0;
+        char            key[256] = {0};
+        uint64_t        sec = 0;
+        int             i = 0;
+        uint64_t        count = 0;
+
+        GF_ASSERT (stats);
+        GF_ASSERT (now);
+        GF_ASSERT (dict);
+        GF_ASSERT (this);
+
+        if (interval == -1)
+                snprintf (key, sizeof (key), "cumulative");
+        else
+                snprintf (key, sizeof (key), "interval");
+        ret = dict_set_int32 (dict, key, interval);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "failed to set "
+                        "interval %d", interval);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-duration", interval);
+        sec = (uint64_t) (now->tv_sec - stats->started_at.tv_sec);
+        ret = dict_set_uint64 (dict, key, sec);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set "
+                        "duration(%d) - %"PRId64, interval, sec);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-total-read", interval);
+        ret = dict_set_uint64 (dict, key, stats->data_read);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
+                       "read(%d) - %"PRId64, interval, stats->data_read);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-total-write", interval);
+        ret = dict_set_uint64 (dict, key, stats->data_written);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
+                        "write(%d) - %"PRId64, interval, stats->data_written);
+                goto out;
+        }
+        for (i = 0; i < 32; i++) {
+                if (stats->block_count_read[i]) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%d-read-%d", interval,
+                                  (1 << i));
+                        count = stats->block_count_read[i];
+                        ret = dict_set_uint64 (dict, key, count);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                        "set read-%db+, with: %"PRId64,
+                                        (1<<i), count);
+                                goto out;
+                        }
+                }
+        }
+
+        for (i = 0; i < 32; i++) {
+                if (stats->block_count_write[i]) {
+                        snprintf (key, sizeof (key), "%d-write-%d", interval,
+                                  (1<<i));
+                        count = stats->block_count_write[i];
+                        ret = dict_set_uint64 (dict, key, count);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                        "set write-%db+, with: %"PRId64,
+                                        (1<<i), count);
+                                goto out;
+                        }
+                }
+        }
+
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                if (stats->fop_hits[i] == 0)
+                        continue;
+                snprintf (key, sizeof (key), "%d-%d-hits", interval, i);
+                ret = dict_set_uint64 (dict, key, stats->fop_hits[i]);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "set %s-fop-hits: %"PRIu64, gf_fop_list[i],
+                                stats->fop_hits[i]);
+                        goto out;
+                }
+
+                if (stats->latency[i].avg == 0)
+                        continue;
+                snprintf (key, sizeof (key), "%d-%d-avglatency", interval, i);
+                ret = dict_set_double (dict, key, stats->latency[i].avg);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
+                                "avglatency(%d) with %f", gf_fop_list[i],
+                                interval, stats->latency[i].avg);
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "%d-%d-minlatency", interval, i);
+                ret = dict_set_double (dict, key, stats->latency[i].min);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
+                                "minlatency(%d) with %f", gf_fop_list[i],
+                                interval, stats->latency[i].min);
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "%d-%d-maxlatency", interval, i);
+                ret = dict_set_double (dict, key, stats->latency[i].max);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
+                                "maxlatency(%d) with %f", gf_fop_list[i],
+                                interval, stats->latency[i].max);
+                        goto out;
+                }
+        }
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+io_stats_dump_global (xlator_t *this, struct ios_global_stats *stats,
+                      struct timeval *now, int interval,
+                      struct ios_dump_args *args)
+{
+        int     ret = -1;
+
+        GF_ASSERT (args);
+        GF_ASSERT (now);
+        GF_ASSERT (stats);
+        GF_ASSERT (this);
+
+
+
+        switch (args->type) {
+        case IOS_DUMP_TYPE_FILE:
+                ret = io_stats_dump_global_to_logfp (this, stats, now,
+                                                     interval, args->u.logfp);
+        break;
+        case IOS_DUMP_TYPE_DICT:
+                ret = io_stats_dump_global_to_dict (this, stats, now,
+                                                    interval, args->u.dict);
+        break;
+        default:
+                GF_ASSERT (0);
+                ret = -1;
+        break;
+        }
+        return ret;
+}
+
+int
+ios_dump_args_init (struct ios_dump_args *args, ios_dump_type_t type,
+                    void *output)
+{
+        int             ret = 0;
+
+        GF_ASSERT (args);
+        GF_ASSERT (type > IOS_DUMP_TYPE_NONE && type < IOS_DUMP_TYPE_MAX);
+        GF_ASSERT (output);
+
+        args->type = type;
+        switch (args->type) {
+        case IOS_DUMP_TYPE_FILE:
+                args->u.logfp = output;
+                break;
+        case IOS_DUMP_TYPE_DICT:
+                args->u.dict = output;
+                break;
+        default:
+                GF_ASSERT (0);
+                ret = -1;
+        }
+
+        return ret;
+}
+
+int
+io_stats_dump (xlator_t *this, struct ios_dump_args *args)
+{
+        struct ios_conf         *conf = NULL;
+        struct ios_global_stats  cumulative = {0, };
+        struct ios_global_stats  incremental = {0, };
+        int                      increment = 0;
+        struct timeval           now;
+
+        GF_ASSERT (this);
+        GF_ASSERT (args);
+        GF_ASSERT (args->type > IOS_DUMP_TYPE_NONE);
+        GF_ASSERT (args->type < IOS_DUMP_TYPE_MAX);
+
+        conf = this->private;
+
+        gettimeofday (&now, NULL);
+        LOCK (&conf->lock);
+        {
+                cumulative  = conf->cumulative;
+                incremental = conf->incremental;
+
+                increment = conf->increment++;
+
+                memset (&conf->incremental, 0, sizeof (conf->incremental));
+                conf->incremental.started_at = now;
+        }
+        UNLOCK (&conf->lock);
+
+        io_stats_dump_global (this, &cumulative, &now, -1, args);
+        io_stats_dump_global (this, &incremental, &now, increment, args);
+
+        return 0;
+}
+
+
+int
+io_stats_dump_fd (xlator_t *this, struct ios_fd *iosfd)
+{
+        struct ios_conf         *conf = NULL;
+        struct timeval           now;
+        uint64_t                 sec = 0;
+        uint64_t                 usec = 0;
+        int                      i = 0;
+
+        conf = this->private;
+
+        if (!conf->dump_fd_stats)
+                return 0;
+
+        if (!iosfd)
+                return 0;
+
+        gettimeofday (&now, NULL);
+
+        if (iosfd->opened_at.tv_usec > now.tv_usec) {
+                now.tv_usec += 1000000;
+                now.tv_usec--;
+        }
+
+        sec = now.tv_sec - iosfd->opened_at.tv_sec;
+        usec = now.tv_usec - iosfd->opened_at.tv_usec;
+
+        gf_log (this->name, GF_LOG_INFO,
+                "--- fd stats ---");
+
+        if (iosfd->filename)
+                gf_log (this->name, GF_LOG_INFO,
+                        "      Filename : %s",
+                        iosfd->filename);
+
+        if (sec)
+                gf_log (this->name, GF_LOG_INFO,
+                        "      Lifetime : %"PRId64"secs, %"PRId64"usecs",
+                        sec, usec);
+
+        if (iosfd->data_read)
+                gf_log (this->name, GF_LOG_INFO,
+                        "     BytesRead : %"PRId64" bytes",
+                        iosfd->data_read);
+
+        if (iosfd->data_written)
+                gf_log (this->name, GF_LOG_INFO,
+                        "  BytesWritten : %"PRId64" bytes",
+                        iosfd->data_written);
+
+        for (i = 0; i < 32; i++) {
+                if (iosfd->block_count_read[i])
+                        gf_log (this->name, GF_LOG_INFO,
+                                " Read %06db+ : %"PRId64,
+                                (1 << i), iosfd->block_count_read[i]);
+        }
+        for (i = 0; i < 32; i++) {
+                if (iosfd->block_count_write[i])
+                        gf_log (this->name, GF_LOG_INFO,
+                                "Write %06db+ : %"PRId64,
+                                (1 << i), iosfd->block_count_write[i]);
+        }
+        return 0;
+}
+
+static void
+update_ios_latency_stats (struct ios_global_stats   *stats, double elapsed,
+                          glusterfs_fop_t op)
+{
+        double avg;
+
+        GF_ASSERT (stats);
+
+        if (!stats->latency[op].min)
+                stats->latency[op].min = elapsed;
+        if (stats->latency[op].min > elapsed)
+                stats->latency[op].min = elapsed;
+        if (stats->latency[op].max < elapsed)
+                stats->latency[op].max = elapsed;
+
+        avg = stats->latency[op].avg;
+
+        stats->latency[op].avg = avg + (elapsed - avg) / stats->fop_hits[op];
+}
+
+int
+update_ios_latency (struct ios_conf *conf, call_frame_t *frame,
+                    glusterfs_fop_t op)
+{
+        double elapsed;
+        struct timeval *begin, *end;
+
+        begin = &frame->begin;
+        end   = &frame->end;
+
+        elapsed = (end->tv_sec - begin->tv_sec) * 1e6
+                + (end->tv_usec - begin->tv_usec);
+
+        update_ios_latency_stats (&conf->cumulative, elapsed, op);
+        update_ios_latency_stats (&conf->incremental, elapsed, op);
+
+        return 0;
+}
+
+int32_t
+io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
+                             ios_stats_type_t flags, int32_t list_cnt)
+{
+        struct ios_conf         *conf = NULL;
+        int                      cnt  = 0;
+        char                     key[256];
+        struct ios_stat_head    *list_head = NULL;
+        struct ios_stat_list    *entry = NULL;
+        int                      ret = -1;
+        ios_stats_thru_t         index = IOS_STATS_THRU_MAX;
+        char                     timestr[256] = {0, };
+        char                    *dict_timestr = NULL;
+
+        conf = this->private;
+
+        switch (flags) {
+                case IOS_STATS_TYPE_OPEN:
+                        list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+                        LOCK (&conf->lock);
+                        {
+                                ret = dict_set_uint64 (resp, "current-open",
+                                                     conf->cumulative.nr_opens);
+                                if (ret)
+                                        goto unlock;
+                                ret = dict_set_uint64 (resp, "max-open",
+                                                       conf->cumulative.max_nr_opens);
+
+                                gf_time_fmt (timestr, sizeof timestr,
+                                             conf->cumulative.max_openfd_time.tv_sec,
+                                             gf_timefmt_FT);
+                                if (conf->cumulative.max_openfd_time.tv_sec)
+                                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                                                  ".%"GF_PRI_SUSECONDS,
+                                                  conf->cumulative.max_openfd_time.tv_usec);
+
+                                dict_timestr = gf_strdup (timestr);
+                                if (!dict_timestr)
+                                        goto unlock;
+                                ret = dict_set_dynstr (resp, "max-openfd-time",
+                                                       dict_timestr);
+                                if (ret)
+                                        goto unlock;
+                        }
+        unlock:
+                        UNLOCK (&conf->lock);
+                        /* Do not proceed if we came here because of some error
+                         * during the dict operation */
+                        if (ret)
+                                goto out;
+                        break;
+                case IOS_STATS_TYPE_READ:
+                        list_head = &conf->list[IOS_STATS_TYPE_READ];
+                        break;
+                case IOS_STATS_TYPE_WRITE:
+                        list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+                        break;
+                case IOS_STATS_TYPE_OPENDIR:
+                        list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+                        break;
+                case IOS_STATS_TYPE_READDIRP:
+                        list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+                        break;
+                case IOS_STATS_TYPE_READ_THROUGHPUT:
+                        list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+                        index = IOS_STATS_THRU_READ;
+                        break;
+                case IOS_STATS_TYPE_WRITE_THROUGHPUT:
+                        list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+                        index = IOS_STATS_THRU_WRITE;
+                        break;
+
+                default:
+                       goto out;
+        }
+        ret = dict_set_int32 (resp, "top-op", flags);
+        if (!list_cnt)
+                goto out;
+        LOCK (&list_head->lock);
+        {
+                list_for_each_entry (entry, &list_head->iosstats->list, list) {
+
+                        cnt++;
+                        snprintf (key, 256, "%s-%d", "filename", cnt);
+                        ret = dict_set_str (resp, key, entry->iosstat->filename);
+                        if (ret)
+                                goto unlock_list_head;
+                         snprintf (key, 256, "%s-%d", "value",cnt);
+                         ret = dict_set_uint64 (resp, key, entry->value);
+                         if (ret)
+                                 goto unlock_list_head;
+                         if (index != IOS_STATS_THRU_MAX) {
+                                 snprintf (key, 256, "%s-%d", "time-sec", cnt);
+                                 ret = dict_set_int32 (resp, key,
+                                         entry->iosstat->thru_counters[index].time.tv_sec);
+                                 if (ret)
+                                         goto unlock_list_head;
+                                 snprintf (key, 256, "%s-%d", "time-usec", cnt);
+                                 ret = dict_set_int32 (resp, key,
+                                         entry->iosstat->thru_counters[index].time.tv_usec);
+                                 if (ret)
+                                         goto unlock_list_head;
+                         }
+                         if (cnt == list_cnt)
+                                 break;
+
+                }
+        }
+unlock_list_head:
+        UNLOCK (&list_head->lock);
+        /* ret is !=0 if some dict operation in the above critical region
+         * failed. */
+        if (ret)
+                goto out;
+        ret = dict_set_int32 (resp, "members", cnt);
+ out:
+        return ret;
+}
+
+int
+io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd,
+                     inode_t *inode, struct iatt *buf,
+                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        struct ios_fd *iosfd = NULL;
+        char          *path = NULL;
+        struct ios_stat *iosstat = NULL;
+        struct ios_conf   *conf = NULL;
+
+        conf = this->private;
+
+        path = frame->local;
+        frame->local = NULL;
+
+        if (!path)
+                goto unwind;
+
+        if (op_ret < 0) {
+                GF_FREE (path);
+                goto unwind;
+        }
+
+        iosfd = GF_CALLOC (1, sizeof (*iosfd), gf_io_stats_mt_ios_fd);
+        if (!iosfd) {
+                GF_FREE (path);
+                goto unwind;
+        }
+
+        iosfd->filename = path;
+        gettimeofday (&iosfd->opened_at, NULL);
+
+        ios_fd_ctx_set (fd, this, iosfd);
+        LOCK (&conf->lock);
+        {
+                conf->cumulative.nr_opens++;
+                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+                        conf->cumulative.max_openfd_time = iosfd->opened_at;
+                }
+        }
+        UNLOCK (&conf->lock);
+
+        iosstat = GF_CALLOC (1, sizeof (*iosstat), gf_io_stats_mt_ios_stat);
+        if (!iosstat) {
+                GF_FREE (path);
+                goto unwind;
+        }
+        iosstat->filename = gf_strdup (path);
+        uuid_copy (iosstat->gfid, buf->ia_gfid);
+        LOCK_INIT (&iosstat->lock);
+        ios_inode_ctx_set (fd->inode, this, iosstat);
+
+unwind:
+        UPDATE_PROFILE_STATS (frame, CREATE);
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        struct ios_fd *iosfd = NULL;
+        char          *path = NULL;
+        struct   ios_stat *iosstat = NULL;
+        struct ios_conf   *conf = NULL;
+
+        conf = this->private;
+        path = frame->local;
+        frame->local = NULL;
+
+        if (!path)
+                goto unwind;
+
+        if (op_ret < 0) {
+                GF_FREE (path);
+                goto unwind;
+        }
+
+        iosfd = GF_CALLOC (1, sizeof (*iosfd), gf_io_stats_mt_ios_fd);
+        if (!iosfd) {
+                GF_FREE (path);
+                goto unwind;
+        }
+
+        iosfd->filename = path;
+        gettimeofday (&iosfd->opened_at, NULL);
+
+        ios_fd_ctx_set (fd, this, iosfd);
+
+        ios_inode_ctx_get (fd->inode, this, &iosstat);
+        if (!iosstat) {
+                iosstat = GF_CALLOC (1, sizeof (*iosstat),
+                                     gf_io_stats_mt_ios_stat);
+                if (iosstat) {
+                        iosstat->filename = gf_strdup (path);
+                        uuid_copy (iosstat->gfid, fd->inode->gfid);
+                        LOCK_INIT (&iosstat->lock);
+                        ios_inode_ctx_set (fd->inode, this, iosstat);
+                }
+        }
+
+        LOCK (&conf->lock);
+        {
+                conf->cumulative.nr_opens++;
+                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+                        conf->cumulative.max_openfd_time = iosfd->opened_at;
+                }
+        }
+        UNLOCK (&conf->lock);
+        if (iosstat) {
+              BUMP_STATS (iosstat, IOS_STATS_TYPE_OPEN);
+              iosstat = NULL;
+        }
+unwind:
+        UPDATE_PROFILE_STATS (frame, OPEN);
+
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+
+}
+
+
+int
+io_stats_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, STAT);
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+
+int
+io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iovec *vector, int32_t count,
+                    struct iatt *buf, struct iobref *iobref, dict_t *xdata)
+{
+        int              len = 0;
+        fd_t            *fd = NULL;
+        struct ios_stat *iosstat = NULL;
+
+        fd = frame->local;
+        frame->local = NULL;
+
+        if (op_ret > 0) {
+                len = iov_length (vector, count);
+                BUMP_READ (fd, len);
+        }
+
+        UPDATE_PROFILE_STATS (frame, READ);
+        ios_inode_ctx_get (fd->inode, this, &iosstat);
+
+        if (iosstat) {
+              BUMP_STATS (iosstat, IOS_STATS_TYPE_READ);
+              BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_READ);
+              iosstat = NULL;
+        }
+
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
+                             vector, count, buf, iobref, xdata);
+        return 0;
+
+}
+
+
+int
+io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        struct ios_stat *iosstat = NULL;
+        inode_t         *inode   = NULL;
+
+        UPDATE_PROFILE_STATS (frame, WRITE);
+        if (frame->local){
+                inode = frame->local;
+                frame->local = NULL;
+                ios_inode_ctx_get (inode, this, &iosstat);
+                if (iosstat) {
+                        BUMP_STATS (iosstat, IOS_STATS_TYPE_WRITE);
+                        BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
+                        inode = NULL;
+                        iosstat = NULL;
+                }
+        }
+
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+
+}
+
+
+
+
+int
+io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+{
+        struct ios_stat *iosstat = NULL;
+        inode_t         *inode   = frame->local;
+
+        frame->local = NULL;
+
+        UPDATE_PROFILE_STATS (frame, READDIRP);
+
+        ios_inode_ctx_get (inode, this, &iosstat);
+
+        if (iosstat) {
+              BUMP_STATS (iosstat, IOS_STATS_TYPE_READDIRP);
+              iosstat = NULL;
+        }
+
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+
+int
+io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, READDIR);
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FSYNC);
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+
+int
+io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, SETATTR);
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
+        return 0;
+}
+
+
+int
+io_stats_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, UNLINK);
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+        return 0;
+
+}
+
+
+int
+io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, RENAME);
+        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
+                             preoldparent, postoldparent,
+                             prenewparent, postnewparent, xdata);
+        return 0;
+}
+
+
+int
+io_stats_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, const char *buf,
+                       struct iatt *sbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, READLINK);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
+        return 0;
+}
+
+
+int
+io_stats_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     inode_t *inode, struct iatt *buf,
+                     dict_t *xdata, struct iatt *postparent)
+{
+        UPDATE_PROFILE_STATS (frame, LOOKUP);
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                             postparent);
+        return 0;
+}
+
+
+int
+io_stats_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      inode_t *inode, struct iatt *buf,
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, SYMLINK);
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+io_stats_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    inode_t *inode, struct iatt *buf,
+                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, MKNOD);
+        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    inode_t *inode, struct iatt *buf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t *xdata)
+{
+        struct ios_stat *iosstat = NULL;
+        char   *path = frame->local;
+
+        UPDATE_PROFILE_STATS (frame, MKDIR);
+        if (op_ret < 0)
+                goto unwind;
+
+        iosstat = GF_CALLOC (1, sizeof (*iosstat), gf_io_stats_mt_ios_stat);
+        if (iosstat) {
+                LOCK_INIT (&iosstat->lock);
+                iosstat->filename = gf_strdup(path);
+                uuid_copy (iosstat->gfid, buf->ia_gfid);
+                ios_inode_ctx_set (inode, this, iosstat);
+        }
+
+unwind:
+        /* local is assigned with path */
+        GF_FREE (frame->local);
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+io_stats_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   inode_t *inode, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, LINK);
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+io_stats_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FLUSH);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        struct ios_stat *iosstat = NULL;
+        int              ret     = -1;
+
+        UPDATE_PROFILE_STATS (frame, OPENDIR);
+        if (op_ret < 0)
+                goto unwind;
+
+        ios_fd_ctx_set (fd, this, 0);
+
+        ret = ios_inode_ctx_get (fd->inode, this, &iosstat);
+        if (!ret)
+                BUMP_STATS (iosstat, IOS_STATS_TYPE_OPENDIR);
+
+unwind:
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+
+int
+io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+
+        UPDATE_PROFILE_STATS (frame, RMDIR);
+
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+
+int
+io_stats_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, TRUNCATE);
+        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
+                             prebuf, postbuf, xdata);
+        return 0;
+}
+
+
+int
+io_stats_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, STATFS);
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+
+int
+io_stats_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, SETXATTR);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+io_stats_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, GETXATTR);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+io_stats_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, REMOVEXATTR);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+io_stats_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FSETXATTR);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FGETXATTR);
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FREMOVEXATTR);
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FSYNCDIR);
+        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, ACCESS);
+        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+io_stats_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FTRUNCATE);
+        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
+                             prebuf, postbuf, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FSTAT);
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		       struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, FALLOCATE);
+	STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+
+int
+io_stats_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		     struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, DISCARD);
+	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+int
+io_stats_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS(frame, ZEROFILL);
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+        return 0;
+}
+
+int
+io_stats_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, LK);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
+        return 0;
+}
+
+
+int
+io_stats_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, ENTRYLK);
+        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+io_stats_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, XATTROP);
+        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FXATTROP);
+        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+io_stats_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, INODELK);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+io_stats_entrylk (call_frame_t *frame, xlator_t *this,
+                  const char *volume, loc_t *loc, const char *basename,
+                  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_entrylk_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->entrylk,
+                    volume, loc, basename, cmd, type, xdata);
+        return 0;
+}
+
+
+int
+io_stats_inodelk (call_frame_t *frame, xlator_t *this,
+                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_inodelk_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->inodelk,
+                    volume, loc, cmd, flock, xdata);
+        return 0;
+}
+
+
+int
+io_stats_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+
+        UPDATE_PROFILE_STATS (frame, FINODELK);
+        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+io_stats_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_finodelk_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->finodelk,
+                    volume, fd, cmd, flock, xdata);
+        return 0;
+}
+
+
+int
+io_stats_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_xattrop_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop,
+                    loc, flags, dict, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_fxattrop_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fxattrop,
+                    fd, flags, dict, xdata);
+        return 0;
+}
+
+
+int
+io_stats_lookup (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_lookup_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    loc, xdata);
+        return 0;
+}
+
+
+int
+io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_stat_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+        return 0;
+}
+
+
+int
+io_stats_readlink (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, size_t size, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_readlink_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readlink,
+                    loc, size, xdata);
+        return 0;
+}
+
+
+int
+io_stats_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_mknod_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mknod,
+                    loc, mode, dev, umask, xdata);
+        return 0;
+}
+
+
+int
+io_stats_mkdir (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+{
+        frame->local = gf_strdup (loc->path);
+
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_mkdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+        return 0;
+}
+
+
+int
+io_stats_unlink (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, int xflag, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_unlink_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+        return 0;
+}
+
+
+int
+io_stats_rmdir (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, int flags, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_rmdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rmdir,
+                    loc, flags, xdata);
+        return 0;
+}
+
+
+int
+io_stats_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_symlink_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->symlink,
+                    linkpath, loc, umask, xdata);
+        return 0;
+}
+
+
+int
+io_stats_rename (call_frame_t *frame, xlator_t *this,
+                 loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_rename_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+
+int
+io_stats_link (call_frame_t *frame, xlator_t *this,
+               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_link_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+
+int
+io_stats_setattr (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_setattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+        return 0;
+}
+
+
+int
+io_stats_truncate (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, off_t offset, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_truncate_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->truncate,
+                    loc, offset, xdata);
+        return 0;
+}
+
+
+int
+io_stats_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int32_t flags, fd_t *fd, dict_t *xdata)
+{
+        frame->local = gf_strdup (loc->path);
+
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_open_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
+        return 0;
+}
+
+
+int
+io_stats_create (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, int32_t flags, mode_t mode,
+                 mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        frame->local = gf_strdup (loc->path);
+
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_create_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+}
+
+
+int
+io_stats_readv (call_frame_t *frame, xlator_t *this,
+                fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        frame->local = fd;
+
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_readv_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+        return 0;
+}
+
+
+int
+io_stats_writev (call_frame_t *frame, xlator_t *this,
+                 fd_t *fd, struct iovec *vector,
+                 int32_t count, off_t offset,
+                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        int                 len = 0;
+
+        if (fd->inode)
+                frame->local = fd->inode;
+        len = iov_length (vector, count);
+
+        BUMP_WRITE (fd, len);
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_writev_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev,
+                    fd, vector, count, offset, flags, iobref, xdata);
+        return 0;
+
+}
+
+
+int
+io_stats_statfs (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_statfs_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->statfs,
+                    loc, xdata);
+        return 0;
+}
+
+
+int
+io_stats_flush (call_frame_t *frame, xlator_t *this,
+                fd_t *fd, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_flush_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->flush,
+                    fd, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fsync (call_frame_t *frame, xlator_t *this,
+                fd_t *fd, int32_t flags, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_fsync_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsync,
+                    fd, flags, xdata);
+        return 0;
+}
+
+
+int
+conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
+{
+        struct {
+                xlator_t       *this;
+                inode_t        *inode;
+                const char     *path;
+        } *stub;
+        xlator_t             *this = NULL;
+        char                 *filename = NULL;
+        FILE                 *logfp = NULL;
+        struct ios_dump_args args = {0};
+
+        stub  = data;
+        this  = stub->this;
+
+        filename = alloca (value->len + 1);
+        memset (filename, 0, value->len + 1);
+        memcpy (filename, data_to_str (value), value->len);
+
+        if (fnmatch ("*io*stat*dump", key, 0) == 0) {
+
+                if (!strncmp (filename, "", 1)) {
+                        gf_log (this->name, GF_LOG_ERROR, "No filename given");
+                        return -1;
+                }
+                logfp = fopen (filename, "w+");
+                if (!logfp) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to open %s "
+                                "for writing", filename);
+                        return -1;
+                }
+                (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_FILE,
+                                           logfp);
+                io_stats_dump (this, &args);
+                fclose (logfp);
+        }
+        return 0;
+}
+
+
+int
+io_stats_setxattr (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, dict_t *dict,
+                   int32_t flags, dict_t *xdata)
+{
+        struct {
+                xlator_t     *this;
+                inode_t      *inode;
+                const char   *path;
+        } stub;
+
+        stub.this  = this;
+        stub.inode = loc->inode;
+        stub.path  = loc->path;
+
+        dict_foreach (dict, conditional_dump, &stub);
+
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_setxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+        return 0;
+}
+
+
+int
+io_stats_getxattr (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, const char *name, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_getxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+
+int
+io_stats_removexattr (call_frame_t *frame, xlator_t *this,
+                      loc_t *loc, const char *name, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_removexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fsetxattr (call_frame_t *frame, xlator_t *this,
+                    fd_t *fd, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_fsetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fgetxattr (call_frame_t *frame, xlator_t *this,
+                    fd_t *fd, const char *name, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_fgetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fremovexattr (call_frame_t *frame, xlator_t *this,
+                       fd_t *fd, const char *name, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_fremovexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+
+int
+io_stats_opendir (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_opendir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+        return 0;
+}
+
+int
+io_stats_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                   off_t offset, dict_t *dict)
+{
+        frame->local = fd->inode;
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_readdirp_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdirp,
+                    fd, size, offset, dict);
+        return 0;
+}
+
+
+int
+io_stats_readdir (call_frame_t *frame, xlator_t *this,
+                  fd_t *fd, size_t size, off_t offset, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_readdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir,
+                    fd, size, offset, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fsyncdir (call_frame_t *frame, xlator_t *this,
+                   fd_t *fd, int32_t datasync, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_fsyncdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsyncdir,
+                    fd, datasync, xdata);
+        return 0;
+}
+
+
+int
+io_stats_access (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, int32_t mask, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_access_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->access,
+                    loc, mask, xdata);
+        return 0;
+}
+
+
+int
+io_stats_ftruncate (call_frame_t *frame, xlator_t *this,
+                    fd_t *fd, off_t offset, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_ftruncate_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate,
+                    fd, offset, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fsetattr (call_frame_t *frame, xlator_t *this,
+                   fd_t *fd, struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_setattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fstat (call_frame_t *frame, xlator_t *this,
+                fd_t *fd, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_fstat_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		   off_t offset, size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		   xdata);
+
+	return 0;
+}
+
+
+int
+io_stats_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+		 size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+
+	return 0;
+}
+
+int
+io_stats_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 size_t len, dict_t *xdata)
+{
+        START_FOP_LATENCY(frame);
+
+        STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+
+        return 0;
+}
+
+
+int
+io_stats_lk (call_frame_t *frame, xlator_t *this,
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_lk_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lk,
+                    fd, cmd, lock, xdata);
+        return 0;
+}
+
+
+int
+io_stats_release (xlator_t *this, fd_t *fd)
+{
+        struct ios_fd  *iosfd = NULL;
+        struct ios_conf *conf = NULL;
+
+        BUMP_FOP (RELEASE);
+
+        conf = this->private;
+
+        LOCK (&conf->lock);
+        {
+                conf->cumulative.nr_opens--;
+        }
+        UNLOCK (&conf->lock);
+
+        ios_fd_ctx_get (fd, this, &iosfd);
+        if (iosfd) {
+                io_stats_dump_fd (this, iosfd);
+
+                GF_FREE (iosfd->filename);
+                GF_FREE (iosfd);
+        }
+
+        return 0;
+}
+
+
+int
+io_stats_releasedir (xlator_t *this, fd_t *fd)
+{
+        BUMP_FOP (RELEASEDIR);
+
+        return 0;
+}
+
+
+int
+io_stats_forget (xlator_t *this, inode_t *inode)
+{
+        BUMP_FOP (FORGET);
+        ios_stats_cleanup (this, inode);
+        return 0;
+}
+
+static int
+ios_init_top_stats (struct ios_conf *conf)
+{
+        int     i = 0;
+
+        GF_ASSERT (conf);
+
+        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
+                conf->list[i].iosstats = GF_CALLOC (1,
+                                         sizeof(*conf->list[i].iosstats),
+                                         gf_io_stats_mt_ios_stat);
+
+                if (!conf->list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
+                LOCK_INIT (&conf->list[i].lock);
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
+                conf->thru_list[i].iosstats = GF_CALLOC (1,
+                                 sizeof (*conf->thru_list[i].iosstats),
+                                 gf_io_stats_mt_ios_stat);
+
+                if (!conf->thru_list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
+                LOCK_INIT (&conf->thru_list[i].lock);
+        }
+
+        return 0;
+}
+
+static void
+ios_destroy_top_stats (struct ios_conf *conf)
+{
+        int                     i = 0;
+        struct ios_stat_head    *list_head = NULL;
+        struct ios_stat_list    *entry     = NULL;
+        struct ios_stat_list    *tmp       = NULL;
+        struct ios_stat_list    *list      = NULL;
+        struct ios_stat         *stat      = NULL;
+
+        GF_ASSERT (conf);
+
+        LOCK (&conf->lock);
+
+        conf->cumulative.nr_opens = 0;
+        conf->cumulative.max_nr_opens = 0;
+        conf->cumulative.max_openfd_time.tv_sec = 0;
+        conf->cumulative.max_openfd_time.tv_usec = 0;
+
+        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+                list_head = &conf->list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+                list_head = &conf->thru_list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        UNLOCK (&conf->lock);
+
+        return;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        struct ios_conf    *conf = NULL;
+        int                 ret = -1;
+        char               *sys_log_str = NULL;
+        int                 sys_log_level = -1;
+        char               *log_str = NULL;
+        int                 log_level = -1;
+
+        if (!this || !this->private)
+                goto out;
+
+        conf = this->private;
+
+        GF_OPTION_RECONF ("dump-fd-stats", conf->dump_fd_stats, options, bool,
+                          out);
+
+        GF_OPTION_RECONF ("count-fop-hits", conf->count_fop_hits, options, bool,
+                          out);
+
+        GF_OPTION_RECONF ("latency-measurement", conf->measure_latency,
+                          options, bool, out);
+
+        GF_OPTION_RECONF ("sys-log-level", sys_log_str, options, str, out);
+        if (sys_log_str) {
+                sys_log_level = glusterd_check_log_level (sys_log_str);
+                set_sys_log_level (sys_log_level);
+        }
+
+        GF_OPTION_RECONF ("log-level", log_str, options, str, out);
+        if (log_str) {
+                log_level = glusterd_check_log_level (log_str);
+                gf_log_set_loglevel (log_level);
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
+        return ret;
+}
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_io_stats_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int
+init (xlator_t *this)
+{
+        struct ios_conf    *conf = NULL;
+        char               *sys_log_str = NULL;
+        int                 sys_log_level = -1;
+        char               *log_str = NULL;
+        int                 log_level = -1;
+        int                 ret = -1;
+
+        if (!this)
+                return -1;
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "io_stats translator requires atleast one subvolume");
+                return -1;
+        }
+
+        if (!this->parents) {
+                /* This is very much valid as io-stats currently is loaded
+                 * on top of volumes on both client and server, hence this is
+                 * not an warning message */
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dangling volume. check volfile ");
+        }
+
+        conf = GF_CALLOC (1, sizeof(*conf), gf_io_stats_mt_ios_conf);
+
+        if (!conf) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Out of memory.");
+                return -1;
+        }
+
+        LOCK_INIT (&conf->lock);
+
+        gettimeofday (&conf->cumulative.started_at, NULL);
+        gettimeofday (&conf->incremental.started_at, NULL);
+
+        ret = ios_init_top_stats (conf);
+        if (ret)
+                return -1;
+
+        GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
+
+        GF_OPTION_INIT ("count-fop-hits", conf->count_fop_hits, bool, out);
+
+        GF_OPTION_INIT ("latency-measurement", conf->measure_latency,
+                          bool, out);
+
+        GF_OPTION_INIT ("sys-log-level", sys_log_str, str, out);
+        if (sys_log_str) {
+                sys_log_level = glusterd_check_log_level (sys_log_str);
+                set_sys_log_level (sys_log_level);
+        }
+
+        GF_OPTION_INIT ("log-level", log_str, str, out);
+        if (log_str) {
+                log_level = glusterd_check_log_level (log_str);
+                gf_log_set_loglevel (log_level);
+        }
+
+        this->private = conf;
+        ret = 0;
+out:
+        return ret;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        struct ios_conf *conf = NULL;
+
+        if (!this)
+                return;
+
+        conf = this->private;
+
+        if (!conf)
+                return;
+        this->private = NULL;
+
+        ios_destroy_top_stats (conf);
+
+        GF_FREE(conf);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "io-stats translator unloaded");
+        return;
+}
+
+int
+notify (xlator_t *this, int32_t event, void *data, ...)
+{
+        int          ret = 0;
+        struct ios_dump_args args = {0};
+        dict_t       *output = NULL;
+        dict_t       *dict = NULL;
+        int32_t       top_op = 0;
+        int32_t       list_cnt = 0;
+        double        throughput = 0;
+        double        time = 0;
+        va_list ap;
+
+        dict = data;
+        va_start (ap, data);
+        output = va_arg (ap, dict_t*);
+        va_end (ap);
+        switch (event) {
+        case GF_EVENT_TRANSLATOR_INFO:
+                ret = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+                if (ret) {
+                         ret = dict_set_int32 (output, "top-op", top_op);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set top-op in dict");
+                                goto out;
+                        }
+                        ios_destroy_top_stats (this->private);
+                        ret = ios_init_top_stats (this->private);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to reset top stats");
+                        ret = dict_set_int32 (output, "stats-cleared",
+                                              ret ? 0 : 1);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set stats-cleared"
+                                        " in dict");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "top-op", &top_op);
+                if (!ret) {
+                        ret = dict_get_int32 (dict, "list-cnt", &list_cnt);
+                        if (top_op > IOS_STATS_TYPE_NONE &&
+                            top_op < IOS_STATS_TYPE_MAX)
+                                ret = io_stats_dump_stats_to_dict (this, output,
+                                                             top_op, list_cnt);
+                        if (top_op == IOS_STATS_TYPE_READ_THROUGHPUT ||
+                                top_op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
+                                ret = dict_get_double (dict, "throughput",
+                                                        &throughput);
+                                if (!ret) {
+                                        ret = dict_get_double (dict, "time",
+                                                                &time);
+                                        if (ret)
+                                                goto out;
+                                        ret = dict_set_double (output,
+                                                "throughput", throughput);
+                                        if (ret)
+                                                goto out;
+                                        ret = dict_set_double (output, "time",
+                                                        time);
+                                        if (ret)
+                                                goto out;
+                                }
+                                ret = 0;
+
+                        }
+                } else {
+                        (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_DICT,
+                                           output);
+                        ret = io_stats_dump (this, &args);
+                }
+                break;
+        default:
+                default_notify (this, event, data);
+                break;
+
+        }
+out:
+        return ret;
+}
+
+struct xlator_fops fops = {
+        .stat        = io_stats_stat,
+        .readlink    = io_stats_readlink,
+        .mknod       = io_stats_mknod,
+        .mkdir       = io_stats_mkdir,
+        .unlink      = io_stats_unlink,
+        .rmdir       = io_stats_rmdir,
+        .symlink     = io_stats_symlink,
+        .rename      = io_stats_rename,
+        .link        = io_stats_link,
+        .truncate    = io_stats_truncate,
+        .open        = io_stats_open,
+        .readv       = io_stats_readv,
+        .writev      = io_stats_writev,
+        .statfs      = io_stats_statfs,
+        .flush       = io_stats_flush,
+        .fsync       = io_stats_fsync,
+        .setxattr    = io_stats_setxattr,
+        .getxattr    = io_stats_getxattr,
+        .removexattr = io_stats_removexattr,
+        .fsetxattr    = io_stats_fsetxattr,
+        .fgetxattr    = io_stats_fgetxattr,
+        .fremovexattr = io_stats_fremovexattr,
+        .opendir     = io_stats_opendir,
+        .readdir     = io_stats_readdir,
+        .readdirp    = io_stats_readdirp,
+        .fsyncdir    = io_stats_fsyncdir,
+        .access      = io_stats_access,
+        .ftruncate   = io_stats_ftruncate,
+        .fstat       = io_stats_fstat,
+        .create      = io_stats_create,
+        .lk          = io_stats_lk,
+        .inodelk     = io_stats_inodelk,
+        .finodelk    = io_stats_finodelk,
+        .entrylk     = io_stats_entrylk,
+        .lookup      = io_stats_lookup,
+        .xattrop     = io_stats_xattrop,
+        .fxattrop    = io_stats_fxattrop,
+        .setattr     = io_stats_setattr,
+        .fsetattr    = io_stats_fsetattr,
+	.fallocate   = io_stats_fallocate,
+	.discard     = io_stats_discard,
+        .zerofill    = io_stats_zerofill,
+};
+
+struct xlator_cbks cbks = {
+        .release     = io_stats_release,
+        .releasedir  = io_stats_releasedir,
+        .forget      = io_stats_forget,
+};
+
+struct volume_options options[] = {
+        { .key  = {"dump-fd-stats"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "If on stats related to file-operations would be "
+                         "tracked inside GlusterFS data-structures."
+        },
+        { .key  = { "latency-measurement" },
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "If on stats related to the latency of each operation "
+                         "would be tracked inside GlusterFS data-structures. "
+        },
+        { .key  = {"count-fop-hits"},
+          .type = GF_OPTION_TYPE_BOOL,
+        },
+        { .key = {"log-level"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
+                     "CRITICAL", "NONE", "TRACE"}
+        },
+
+        /* These are synthetic entries to assist validation of CLI's  *
+         *  volume set  command                                       */
+        { .key = {"client-log-level"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "INFO",
+          .description = "Changes the log-level of the clients",
+          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
+                     "CRITICAL", "NONE", "TRACE"}
+        },
+        { .key = {"sys-log-level"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "CRITICAL",
+          .description = "Gluster's syslog log-level",
+          .value = { "WARNING", "ERROR", "INFO", "CRITICAL"}
+        },
+        { .key = {"brick-log-level"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "INFO",
+          .description = "Changes the log-level of the bricks",
+          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
+                     "CRITICAL", "NONE", "TRACE"}
+        },
+        { .key  = {NULL} },
+
+};
diff --git a/xlators/debug/trace/src/Makefile.am b/xlators/debug/trace/src/Makefile.am
index 0f1679a04..7b2597b4d 100644
--- a/xlators/debug/trace/src/Makefile.am
+++ b/xlators/debug/trace/src/Makefile.am
@@ -2,13 +2,15 @@
 xlator_LTLIBRARIES = trace.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-trace_la_LDFLAGS = -module -avoidversion
+trace_la_LDFLAGS = -module -avoid-version
 
 trace_la_SOURCES = trace.c
 trace_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = trace.h trace-mem-types.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/trace/src/trace-mem-types.h b/xlators/debug/trace/src/trace-mem-types.h
new file mode 100644
index 000000000..9fa7d97c2
--- /dev/null
+++ b/xlators/debug/trace/src/trace-mem-types.h
@@ -0,0 +1,21 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __TRACE_MEM_TYPES_H__
+#define __TRACE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_trace_mem_types_ {
+        gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
+        gf_trace_mt_end
+};
+#endif
diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c
index 3ccf11a83..c9d839356 100644
--- a/xlators/debug/trace/src/trace.c
+++ b/xlators/debug/trace/src/trace.c
@@ -1,2321 +1,3274 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include "trace.h"
+#include "trace-mem-types.h"
 
 /**
  * xlators/debug/trace :
- *    This translator logs all the arguments to the fops/mops and also 
- *    their _cbk functions, which later passes the call to next layer. 
+ *    This translator logs all the arguments to the fops/mops and also
+ *    their _cbk functions, which later passes the call to next layer.
  *    Very helpful translator for debugging.
  */
 
-#include <time.h>
-#include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "common-utils.h"
-
-#define ERR_EINVAL_NORETURN(cond)                \
-do                                               \
-  {						 \
-    if ((cond))					 \
-      {						 \
-	gf_log ("ERROR", GF_LOG_ERROR,   	 \
-		"%s: %s: (%s) is true", 	 \
-		__FILE__, __FUNCTION__, #cond);	 \
-      }                                          \
-  } while (0)
-
-typedef struct trace_private {
-	int32_t debug_flag;
-} trace_private_t;
-
-struct {
-	char *name;
-	int enabled;
-} trace_fop_names[GF_FOP_MAXVALUE];
-
-int32_t 
-trace_create_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd,
-		  inode_t *inode,
-		  struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this);
-
-	if (trace_fop_names[GF_FOP_CREATE].enabled) {
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, fd=%p, ino=%"PRIu64"), "
-				"*buf {st_dev=%"GF_PRI_DEV", st_ino=%"PRIu64", "
-				"st_mode=%d, st_nlink=%"GF_PRI_NLINK", st_uid=%d, "
-				"st_gid=%d, st_rdev=%"GF_PRI_DEV", st_size=%"PRId64", "
-				"st_blksize=%"GF_PRI_BLKSIZE", st_blocks=%"PRId64", "
-				"st_atime=%s, st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, fd, inode->ino, buf->st_dev, 
-				buf->st_ino, buf->st_mode, buf->st_nlink, 
-				buf->st_uid, buf->st_gid, buf->st_rdev, buf->st_size, 
-				buf->st_blksize, 
-				buf->st_blocks, atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
-	return 0;
-}
-
-int32_t 
-trace_open_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd)
-{
-	ERR_EINVAL_NORETURN (!this);
-
-	if (trace_fop_names[GF_FOP_OPEN].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d, *fd=%p)",
-			frame->root->unique, op_ret, op_errno, fd);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-int32_t 
-trace_stat_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this);
-  
-	if (trace_fop_names[GF_FOP_STAT].enabled) {
-
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, buf {st_dev=%"GF_PRI_DEV", "
-				"st_ino=%"PRIu64", st_mode=%d, st_nlink=%"GF_PRI_NLINK", "
-				"st_uid=%d, st_gid=%d, st_rdev=%"GF_PRI_DEV", st_size=%"PRId64
-				", st_blksize=%"GF_PRI_BLKSIZE", st_blocks=%"PRId64", "
-				"st_atime=%s, st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, buf->st_dev, buf->st_ino, 
-				buf->st_mode, buf->st_nlink, buf->st_uid, buf->st_gid, 
-				buf->st_rdev, buf->st_size, buf->st_blksize, 
-				buf->st_blocks, atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}    
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_readv_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iovec *vector,
-		 int32_t count,
-		 struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this);
-
-	if (trace_fop_names[GF_FOP_READ].enabled) {
-
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_dev=%"GF_PRI_DEV", "
-				"st_ino=%"PRIu64", st_mode=%d, st_nlink=%"GF_PRI_NLINK", "
-				"st_uid=%d, st_gid=%d, st_rdev=%"GF_PRI_DEV", "
-				"st_size=%"PRId64", st_blksize=%"GF_PRI_BLKSIZE", "
-				"st_blocks=%"PRId64", st_atime=%s, st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, buf->st_dev, buf->st_ino, 
-				buf->st_mode, buf->st_nlink, buf->st_uid, buf->st_gid, 
-				buf->st_rdev, buf->st_size, buf->st_blksize, buf->st_blocks, 
-				atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-  
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, buf);
-	return 0;
-}
-
-int32_t 
-trace_writev_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this);
-
-	if (trace_fop_names[GF_FOP_WRITE].enabled) {
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_ino=%"PRIu64", "
-				"st_size=%"PRId64", st_blocks=%"PRId64", st_atime=%s, "
-				"st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, buf->st_ino, buf->st_size, 
-				buf->st_blocks, atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_getdents_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dir_entry_t *entries,
-		    int32_t count)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_GETDENTS].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d, count=%d)",
-			frame->root->unique, op_ret, op_errno, count);
-	}
-  
-	STACK_UNWIND (frame, op_ret, op_errno, entries, count);
-	return 0;
-}
-
-int32_t 
-trace_readdir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   gf_dirent_t *buf)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_READDIR].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64" :(op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
-  
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-int32_t 
-trace_fsync_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t 
-trace_chown_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_CHOWN].enabled) {
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-    
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_ino=%"PRIu64", st_mode=%d, "
-				"st_uid=%d, st_gid=%d, st_atime=%s, st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, buf->st_ino, buf->st_mode, 
-				buf->st_uid, buf->st_gid, atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_chmod_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_CHMOD].enabled) {
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-    
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_ino=%"PRIu64", st_mode=%d, "
-				"st_atime=%s, st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, buf->st_ino, buf->st_mode, 
-				atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_fchmod_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_FCHMOD].enabled) {
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-    
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_ino=%"PRIu64", st_mode=%d, "
-				"st_atime=%s, st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, buf->st_ino, buf->st_mode, 
-				atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_fchown_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_FCHOWN].enabled) {  
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-    
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_ino=%"PRIu64", st_mode=%d, "
-				"st_uid=%d, st_gid=%d, st_atime=%s, st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, buf->st_ino, buf->st_mode, 
-				buf->st_uid, buf->st_gid, atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_unlink_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t 
-trace_rename_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct stat *buf)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_RENAME].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d, buf {st_ino=%"PRIu64"})",
-			frame->root->unique, op_ret, op_errno, 
-			(buf? buf->st_ino : 0));
-	}
-  
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_readlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    const char *buf)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_READLINK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d, buf=%s)",
-			frame->root->unique, op_ret, op_errno, buf);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_lookup_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-		  struct stat *buf,
-		  dict_t *xattr)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-		if (op_ret >= 0) {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, ino=%"PRIu64", "
-				"*buf {st_dev=%"GF_PRI_DEV", st_ino=%"PRIu64", st_mode=%d, "
-				"st_nlink=%"GF_PRI_NLINK", st_uid=%d, st_gid=%d, "
-				"st_rdev=%"GF_PRI_DEV", st_size=%"PRId64", "
-				"st_blksize=%"GF_PRI_BLKSIZE", st_blocks=%"PRId64"})",
-				frame->root->unique, op_ret, inode->ino, buf->st_dev, buf->st_ino, 
-				buf->st_mode, buf->st_nlink, buf->st_uid, buf->st_gid, 
-				buf->st_rdev, buf->st_size, buf->st_blksize, buf->st_blocks);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, xattr);
-	return 0;
-}
-
-int32_t 
-trace_symlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-		   struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-    
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, ino=%"PRIu64", *buf {st_ino=%"PRIu64", "
-				"st_mode=%d, st_nlink=%"GF_PRI_NLINK", st_uid=%d, st_gid=%d, "
-				"st_size=%"PRId64", st_blocks=%"PRId64", st_atime=%s, "
-				"st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, inode->ino, buf->st_ino, 
-				buf->st_mode, buf->st_nlink, buf->st_uid, buf->st_gid, 
-				buf->st_size, buf->st_blocks, atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-
-int32_t 
-trace_mknod_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-		 struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_MKNOD].enabled) {  
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-    
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, ino=%"PRIu64", *buf {st_dev=%"GF_PRI_DEV
-				", st_ino=%"PRIu64", st_mode=%d, st_nlink=%"GF_PRI_NLINK", "
-				"st_uid=%d, st_gid=%d, st_rdev=%"GF_PRI_DEV", st_size=%"PRId64
-				", st_blksize=%"GF_PRI_BLKSIZE", st_blocks=%"PRId64", st_atime=%s, "
-				"st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, inode->ino, buf->st_dev, buf->st_ino, 
-				buf->st_mode, buf->st_nlink, buf->st_uid, buf->st_gid, 
-				buf->st_rdev, buf->st_size, buf->st_blksize, buf->st_blocks, 
-				atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-  
-
-int32_t 
-trace_mkdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-		 struct stat *buf)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_MKDIR].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d, ino=%"PRIu64"",
-			frame->root->unique, op_ret, op_errno, 
-			(inode? inode->ino : 0));
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-  
-int32_t 
-trace_link_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-		struct stat *buf)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_LINK].enabled) {
-		if (op_ret >= 0) {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, ino=%"PRIu64", "
-				"*buf {st_nlink=%"GF_PRI_NLINK"})",
-				frame->root->unique, op_ret, inode->ino, buf->st_nlink);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-
-int32_t 
-trace_flush_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-int32_t 
-trace_opendir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d, fd=%p)",
-			frame->root->unique, op_ret, op_errno, fd);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-int32_t 
-trace_rmdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t 
-trace_truncate_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct stat *buf)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {  
-		if (op_ret >= 0) {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_size=%"PRId64", st_blksize=%"
-				GF_PRI_BLKSIZE", st_blocks=%"PRId64"})",
-				frame->root->unique, op_ret, buf->st_size, buf->st_blksize, 
-				buf->st_blocks);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_utimens_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_UTIMENS].enabled) {
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-    
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_atime=%s, st_mtime=%s, "
-				"st_ctime=%s})",
-				frame->root->unique, op_ret, atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_statfs_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct statvfs *buf)
-{
-	ERR_EINVAL_NORETURN (!this);
-
-	if (trace_fop_names[GF_FOP_STATFS].enabled) {
-		if (op_ret >= 0) {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": ({f_bsize=%lu, f_frsize=%lu, f_blocks=%"GF_PRI_FSBLK
-				", f_bfree=%"GF_PRI_FSBLK", f_bavail=%"GF_PRI_FSBLK", "
-				"f_files=%"GF_PRI_FSBLK", f_ffree=%"GF_PRI_FSBLK", f_favail=%"
-				GF_PRI_FSBLK", f_fsid=%lu, f_flag=%lu, f_namemax=%lu}) => ret=%d",
-				frame->root->unique, buf->f_bsize, buf->f_frsize, buf->f_blocks, 
-				buf->f_bfree, buf->f_bavail, buf->f_files, buf->f_ffree, 
-				buf->f_favail, buf->f_fsid, buf->f_flag, buf->f_namemax, op_ret);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t 
-trace_getxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dict_t *dict)
-{
-	ERR_EINVAL_NORETURN (!this || !dict);
-
-	if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d, dict=%p)",
-			frame->root->unique, op_ret, op_errno, dict);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-int32_t 
-trace_removexattr_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-int32_t 
-trace_fsyncdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t 
-trace_access_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t 
-trace_ftruncate_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct stat *buf)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-		if (op_ret >= 0) {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_size=%"PRId64", "
-				"st_blksize=%"GF_PRI_BLKSIZE", st_blocks=%"PRId64"})",
-				frame->root->unique, op_ret, buf->st_size, buf->st_blksize, 
-				buf->st_blocks);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_fstat_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct stat *buf)
-{
-	char atime_buf[256], mtime_buf[256], ctime_buf[256];
-	ERR_EINVAL_NORETURN (!this );
-  
-	if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-		if (op_ret >= 0) {
-			strftime (atime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_atime));
-			strftime (mtime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_mtime));
-			strftime (ctime_buf, 256, "[%b %d %H:%M:%S]", localtime (&buf->st_ctime));
-    
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, *buf {st_dev=%"GF_PRI_DEV", "
-				"st_ino=%"PRIu64", st_mode=%d, st_nlink=%"GF_PRI_NLINK", "
-				"st_uid=%d, st_gid=%d, st_rdev=%"GF_PRI_DEV", st_size=%"PRId64", "
-				"st_blksize=%"GF_PRI_BLKSIZE", st_blocks=%"PRId64", st_atime=%s, "
-				"st_mtime=%s, st_ctime=%s})",
-				frame->root->unique, op_ret, buf->st_dev, buf->st_ino, 
-				buf->st_mode, buf->st_nlink, buf->st_uid, buf->st_gid, 
-				buf->st_rdev, buf->st_size, buf->st_blksize, 
-				buf->st_blocks, atime_buf, mtime_buf, ctime_buf);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-trace_lk_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct flock *lock)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_LK].enabled) {
-		if (op_ret >= 0) {
-			gf_log (this->name, GF_LOG_NORMAL,
-				"%"PRId64": (op_ret=%d, {l_type=%d, l_whence=%d, "
-				"l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-				frame->root->unique, op_ret, lock->l_type, lock->l_whence, 
-				lock->l_start, lock->l_len, lock->l_pid);
-		} else {
-			gf_log (this->name, GF_LOG_NORMAL, 
-				"%"PRId64": (op_ret=%d, op_errno=%d)",
-				frame->root->unique, op_ret, op_errno);
-		}    
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, lock);
-	return 0;
-}
-
-
-int32_t 
-trace_setdents_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_SETDENTS].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL,
-			"%"PRId64": op_ret=%d, op_errno=%d",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t 
-trace_entrylk_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno)
-{
-	ERR_EINVAL_NORETURN (!this );
-
-	if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL,
-			"%"PRId64": op_ret=%d, op_errno=%d",
-			frame->root->unique, op_ret, op_errno);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-int32_t 
-trace_xattrop_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   dict_t *dict)
-{
-	ERR_EINVAL_NORETURN (!this || !dict);
-
-	if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
+int
+dump_history_trace (circular_buffer_t *cb, void *data)
+{
+        char    *string      = NULL;
+        struct  tm      *tm  = NULL;
+        char    timestr[256] = {0,};
+
+        string = (char *)cb->data;
+        tm    = localtime (&cb->tv.tv_sec);
+
+        /* Since we are continuing with adding entries to the buffer even when
+           gettimeofday () fails, it's safe to check tm and then dump the time
+           at which the entry was added to the buffer */
+        if (tm) {
+                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
+                gf_proc_dump_write ("TIME", "%s", timestr);
+        }
+
+        gf_proc_dump_write ("FOP", "%s\n", string);
+
+        return 0;
+}
+
+int
+trace_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd,
+                  inode_t *inode, struct iatt *buf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_CREATE].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret >= 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
+                                  "*stbuf {%s}, *preparent {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid), op_ret, fd,
+                                  statstr, preparentstr, postparentstr);
+
+                        /* for 'release' log */
+                        fd_ctx_set (fd, this, 0);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
+                            preparent, postparent, xdata);
+        return 0;
+}
+
+int
+trace_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        trace_conf_t      *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_OPEN].enabled) {
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, "
+                          "*fd=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          fd);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        /* for 'release' log */
+        if (op_ret >= 0)
+                fd_ctx_set (fd, this, 0);
+
+        TRACE_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int
+trace_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
+{
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_STAT].enabled) {
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int
+trace_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                 int32_t count, struct iatt *buf, struct iobref *iobref,
+                 dict_t *xdata)
+{
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READ].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret >= 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
+                            buf, iobref, xdata);
+        return 0;
+}
+
+int
+trace_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        char         preopstr[4096]  = {0, };
+        char         postopstr[4096] = {0, };
+        trace_conf_t *conf           = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_WRITE].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret >= 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+        return 0;
+}
+
+int
+trace_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                   dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READDIR].enabled) {
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (readdir, frame, op_ret, op_errno, buf, xdata);
+
+        return 0;
+}
+
+int
+trace_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                    dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READDIRP].enabled) {
+                char        string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int
+trace_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s}",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+
+        return 0;
+}
+
+int
+trace_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
+                            statpost, xdata);
+        return 0;
+}
+
+int
+trace_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+{
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, uuid_utoa (frame->local),
+                                  op_ret, op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
+                            statpre, statpost, xdata);
+        return 0;
+}
+
+int
+trace_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  " *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr,
+                                  postparentstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
+        return 0;
+}
+
+int
+trace_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  struct iatt *preoldparent, struct iatt *postoldparent,
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
+{
+        char           statstr[4096]          = {0, };
+        char           preoldparentstr[4096]  = {0, };
+        char           postoldparentstr[4096] = {0, };
+        char           prenewparentstr[4096]  = {0, };
+        char           postnewparentstr[4096] = {0, };
+        trace_conf_t   *conf                  = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RENAME].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preoldparent, preoldparentstr);
+                        trace_stat_to_str (postoldparent, postoldparentstr);
+                        trace_stat_to_str (prenewparent, prenewparentstr);
+                        trace_stat_to_str (postnewparent, postnewparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s}, *preoldparent = {%s},"
+                                  " *postoldparent = {%s}"
+                                  " *prenewparent = {%s}, "
+                                  "*postnewparent = {%s})",
+                                  frame->root->unique, op_ret, statstr,
+                                  preoldparentstr, postoldparentstr,
+                                  prenewparentstr, postnewparentstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
+
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
+                            preoldparent, postoldparent,
+                            prenewparent, postnewparent, xdata);
+        return 0;
+}
+
+int
+trace_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    const char *buf, struct iatt *stbuf, dict_t *xdata)
+{
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (stbuf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d,"
+                                  "buf=%s, stbuf = { %s })",
+                                  frame->root->unique, op_ret, op_errno,
+                                  buf, statstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, stbuf,
+                            xdata);
+        return 0;
+}
+
+int
+trace_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  inode_t *inode, struct iatt *buf,
+                  dict_t *xdata, struct iatt *postparent)
+{
+        char          statstr[4096]       = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (postparent, postparentstr);
+                        /* print buf->ia_gfid instead of inode->gfid,
+                         * since if the inode is not yet linked to the
+                         * inode table (fresh lookup) then null gfid
+                         * will be printed.
+                         */
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*buf {%s}, *postparent {%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (buf->ia_gfid),
+                                  op_ret, statstr, postparentstr);
+
+                        /* For 'forget' */
+                        inode_ctx_put (inode, this, 0);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+                            xdata, postparent);
+        return 0;
+}
+
+int
+trace_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   inode_t *inode, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata)
+{
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, op_errno=%d",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
+        return 0;
+}
+
+int
+trace_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
+        return 0;
+}
+
+int
+trace_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  ", *stbuf = {%s}, *prebuf = {%s}, "
+                                  "*postbuf = {%s} )",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
+        return 0;
+}
+
+int
+trace_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno,
+                inode_t *inode, struct iatt *buf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char  string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_LINK].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s},  *prebuf = {%s},"
+                                  " *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  statstr, preparentstr, postparentstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
+        return 0;
+}
+
+int
+trace_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char      string[4096] = {0,};
+        if (trace_fop_names[GF_FOP_FLUSH].enabled) {
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+trace_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char    string[4096] = {0,};
+        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " fd=%p",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno, fd);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        /* for 'releasedir' log */
+        if (op_ret >= 0)
+                fd_ctx_set (fd, this, 0);
+
+        TRACE_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int
+trace_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+        char           preparentstr[4096]  = {0, };
+        char           postparentstr[4096] = {0, };
+        trace_conf_t   *conf               = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "*prebuf={%s},  *postbuf={%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr, postparentstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
+        return 0;
+}
+
+int
+trace_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno,
+                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        char           preopstr[4096]  = {0, };
+        char           postopstr[4096] = {0, };
+        trace_conf_t   *conf           = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
+                            postbuf, xdata);
+        return 0;
+}
+
+int
+trace_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                  dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_STATFS].enabled) {
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": ({f_bsize=%lu, "
+                                  "f_frsize=%lu, "
+                                  "f_blocks=%"GF_PRI_FSBLK
+                                  ", f_bfree=%"GF_PRI_FSBLK", "
+                                  "f_bavail=%"GF_PRI_FSBLK", "
+                                  "f_files=%"GF_PRI_FSBLK", "
+                                  "f_ffree=%"GF_PRI_FSBLK", "
+                                  "f_favail=%"GF_PRI_FSBLK", "
+                                  "f_fsid=%lu, f_flag=%lu, "
+                                  "f_namemax=%lu}) => ret=%d",
+                                  frame->root->unique, buf->f_bsize,
+                                  buf->f_frsize, buf->f_blocks,
+                                  buf->f_bfree, buf->f_bavail,
+                                  buf->f_files, buf->f_ffree,
+                                  buf->f_favail, buf->f_fsid,
+                                  buf->f_flag, buf->f_namemax, op_ret);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int
+trace_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret,
+                          op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+trace_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+
+        return 0;
+}
+
+int
+trace_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
+        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+trace_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+                char      string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+
+        return 0;
+}
+
+int
+trace_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int
+trace_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
+                char       string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+trace_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_ACCESS].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, "
+                          "op_errno=%d)", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+trace_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        char          prebufstr[4096]  = {0, };
+        char          postbufstr[4096] = {0, };
+        trace_conf_t  *conf            = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, prebufstr);
+                        trace_stat_to_str (postbuf, postbufstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  prebufstr, postbufstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+        return 0;
+}
+
+int
+trace_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
+                char string[4096]  = {0.};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d "
+                                  "buf=%s", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int
+trace_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+              dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LK].enabled) {
+                char      string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "{l_type=%d, l_whence=%d, "
+                                  "l_start=%"PRId64", "
+                                  "l_len=%"PRId64", l_pid=%u})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, lock->l_type, lock->l_whence,
+                                  lock->l_start, lock->l_len,
+                                  lock->l_pid);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
+        return 0;
+}
+
+int
+trace_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
+                char   string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+trace_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+trace_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_XATTROP].enabled) {
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+int
+trace_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
-int32_t 
-trace_fxattrop_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dict_t *dict)
+int
+trace_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	ERR_EINVAL_NORETURN (!this || !dict);
+        trace_conf_t   *conf = NULL;
 
-	if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (op_ret=%d, op_errno=%d)",
-			frame->root->unique, op_ret, op_errno);
-	}
+        conf = this->private;
 
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_INODELK].enabled) {
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local),op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
-int32_t 
-trace_inodelk_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno)
+int
+trace_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	ERR_EINVAL_NORETURN (!this );
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
 
-	if (trace_fop_names[GF_FOP_INODELK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL,
-			"%"PRId64": op_ret=%d, op_errno=%d",
-			frame->root->unique, op_ret, op_errno);
-	}
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
+int
+trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     uint32_t weak_checksum, uint8_t *strong_checksum,
+                     dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
 
-int32_t
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
+        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        TRACE_STACK_UNWIND (rchecksum, frame, op_ret, op_errno, weak_checksum,
+                            strong_checksum, xdata);
+
+        return 0;
+}
+
+/* *_cbk section over <----------> fop section start */
+
+int
 trace_entrylk (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, const char *basename,
-	       entrylk_cmd cmd, entrylk_type type)
+               const char *volume, loc_t *loc, const char *basename,
+               entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          volume, loc->path, basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_entrylk_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->entrylk,
+                    volume, loc, basename, cmd, type, xdata);
+        return 0;
+}
+
+int
+trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+               loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+        char         *cmd_str  = NULL;
+        char         *type_str = NULL;
+        trace_conf_t *conf     = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_INODELK].enabled) {
+                char string[4096]  = {0,};
+                switch (cmd) {
+#if F_GETLK != F_GETLK64
+                case F_GETLK64:
+#endif
+                case F_GETLK:
+                        cmd_str = "GETLK";
+                        break;
+
+#if F_SETLK != F_SETLK64
+                case F_SETLK64:
+#endif
+                case F_SETLK:
+                        cmd_str = "SETLK";
+                        break;
+
+#if F_SETLKW != F_SETLKW64
+                case F_SETLKW64:
+#endif
+                case F_SETLKW:
+                        cmd_str = "SETLKW";
+                        break;
+
+                default:
+                        cmd_str = "UNKNOWN";
+                        break;
+                }
+
+                switch (flock->l_type) {
+                case F_RDLCK:
+                        type_str = "READ";
+                        break;
+                case F_WRLCK:
+                        type_str = "WRITE";
+                        break;
+                case F_UNLCK:
+                        type_str = "UNLOCK";
+                        break;
+                default:
+                        type_str = "UNKNOWN";
+                        break;
+                }
+
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), volume,
+                          loc->path, cmd_str, type_str,
+                          (unsigned long long)flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_inodelk_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->inodelk,
+                    volume, loc, cmd, flock, xdata);
+        return 0;
+}
+
+int
+trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+                fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+        char          *cmd_str  = NULL;
+        char          *type_str = NULL;
+        trace_conf_t  *conf     = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+                char  string[4096] = {0,};
+                switch (cmd) {
+#if F_GETLK != F_GETLK64
+                case F_GETLK64:
+#endif
+                case F_GETLK:
+                        cmd_str = "GETLK";
+                        break;
+
+#if F_SETLK != F_SETLK64
+                case F_SETLK64:
+#endif
+                case F_SETLK:
+                        cmd_str = "SETLK";
+                        break;
+
+#if F_SETLKW != F_SETLKW64
+                case F_SETLKW64:
+#endif
+                case F_SETLKW:
+                        cmd_str = "SETLKW";
+                        break;
+
+                default:
+                        cmd_str = "UNKNOWN";
+                        break;
+                }
+
+                switch (flock->l_type) {
+                case F_RDLCK:
+                        type_str = "READ";
+                        break;
+                case F_WRLCK:
+                        type_str = "WRITE";
+                        break;
+                case F_UNLCK:
+                        type_str = "UNLOCK";
+                        break;
+                default:
+                        type_str = "UNKNOWN";
+                        break;
+                }
+
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd =%p "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          cmd_str, type_str,
+                          (unsigned long long) flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        STACK_WIND (frame, trace_finodelk_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->finodelk,
+                    volume, fd, cmd, flock, xdata);
+        return 0;
+}
+
+int
+trace_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	ERR_EINVAL_NORETURN (!this || !loc || !basename);
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_XATTROP].enabled) {
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s (path=%s flags=%d)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_xattrop_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop,
+                    loc, flags, dict, xdata);
 
-	if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc= {path=%s, ino=%"PRIu64"} basename=%s, cmd=%s, type=%s)",
-			frame->root->unique, loc->path, loc->inode->ino, basename, 
-			((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"), 
-			((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
-	}
+        return 0;
+}
+
+int
+trace_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
 
-	STACK_WIND (frame, 
-		    trace_entrylk_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->entrylk,
-		    loc, basename, cmd, type);
-	return 0;
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_fxattrop_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fxattrop,
+                    fd, flags, dict, xdata);
+
+        return 0;
 }
 
-int32_t
-trace_inodelk (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc, int32_t cmd, struct flock *flock)
+int
+trace_lookup (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, dict_t *xdata)
+{
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char string[4096] = {0,};
+                /* TODO: print all the keys mentioned in xattr_req */
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_lookup_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    loc, xdata);
+
+        return 0;
+}
+
+int
+trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
+        if (trace_fop_names[GF_FOP_STAT].enabled) {
+                char  string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_stat_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+
+        return 0;
+}
+
+int
+trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "size=%"GF_PRI_SIZET")", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          size);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_readlink_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readlink,
+                    loc, size, xdata);
+
+        return 0;
+}
+
+int
+trace_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
+        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d "
+                          "umask=0%o, dev=%"GF_PRI_DEV")",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask, dev);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_mknod_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mknod,
+                    loc, mode, dev, umask, xdata);
+
+        return 0;
+}
+
+int
+trace_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_mkdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+        return 0;
+}
+
+int
+trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flag=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          xflag);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        STACK_WIND (frame, trace_unlink_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+        return 0;
+}
+
+int
+trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+             dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_rmdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rmdir,
+                    loc, flags, xdata);
+
+        return 0;
+}
+
+int
+trace_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+               loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+                char     string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s linkpath=%s, path=%s"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), linkpath,
+                          loc->path, umask);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_symlink_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->symlink,
+                    linkpath, loc, umask, xdata);
+
+        return 0;
+}
+
+int
+trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
+{
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RENAME].enabled) {
+                char string[4096] = {0,};
+                if (newloc->inode)
+                        uuid_utoa_r (newloc->inode->gfid, newgfid);
+                else
+                        strcpy (newgfid, "0");
+
+                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s",
+                          frame->root->unique, oldgfid,
+                          oldloc->path, newgfid, newloc->path);
+
+                frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_rename_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+
+        return 0;
+}
+
+int
+trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
+{
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LINK].enabled) {
+                char string[4096]  = {0,};
+                if (newloc->inode)
+                        uuid_utoa_r (newloc->inode->gfid, newgfid);
+                else
+                        strcpy (newgfid, "0");
+
+                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s", frame->root->unique,
+                          oldgfid, oldloc->path, newgfid,
+                          newloc->path);
+
+                frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_link_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+int
+trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        uint64_t     ia_time          = 0;
+        char         actime_str[256]  = {0,};
+        char         modtime_str[256] = {0,};
+        trace_conf_t *conf            = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char     string[4096]  =  {0,};
+                if (valid & GF_SET_ATTR_MODE) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s mode=%o)",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
+                }
+
+                if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s uid=%o,"
+                                  " gid=%o", frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, stbuf->ia_uid,
+                                  stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
+                }
+
+                if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+                        ia_time = stbuf->ia_atime;
+                        strftime (actime_str, 256, "[%b %d %H:%M:%S]",
+                                  localtime ((time_t *)&ia_time));
+
+                        ia_time = stbuf->ia_mtime;
+                        strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
+                                  localtime ((time_t *)&ia_time));
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
+                }
+                frame->local = loc->inode->gfid;
+        }
+
+out:
+        STACK_WIND (frame, trace_setattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+
+        return 0;
+}
+
+int
+trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        uint64_t       ia_time          = 0;
+        char           actime_str[256]  = {0,};
+        char           modtime_str[256] = {0,};
+        trace_conf_t   *conf            = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char     string[4096]  =  {0,};
+                if (valid & GF_SET_ATTR_MODE) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, mode=%o",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid), fd,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
+                }
+
+                if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, uid=%o, "
+                                  "gid=%o", frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, stbuf->ia_uid, stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
+                }
+
+                if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+                        ia_time = stbuf->ia_atime;
+                        strftime (actime_str, 256, "[%b %d %H:%M:%S]",
+                                  localtime ((time_t *)&ia_time));
+
+                        ia_time = stbuf->ia_mtime;
+                        strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
+                                  localtime ((time_t *)&ia_time));
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
+                }
+                frame->local = fd->inode->gfid;
+        }
+
+out:
+        STACK_WIND (frame, trace_fsetattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+
+        return 0;
+}
+
+int
+trace_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                off_t offset, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "offset=%"PRId64"", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          offset);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_truncate_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->truncate,
+                    loc, offset, xdata);
+
+        return 0;
+}
+
+int
+trace_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            int32_t flags, fd_t *fd, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_OPEN].enabled) {
+                char      string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags, fd);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_open_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
+        return 0;
+}
+
+int
+trace_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+              dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_CREATE].enabled) {
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, fd=%p, "
+                          "flags=0%o mode=0%o umask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          fd, flags, mode, umask);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_create_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+}
+
+int
+trace_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READ].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"
+                          GF_PRI_SIZET"offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, flags);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_readv_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+        return 0;
+}
+
+int
+trace_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int32_t count,
+              off_t offset, uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_WRITE].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, count=%d, "
+                          " offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, count,
+                          offset, flags);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_writev_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev,
+                    fd, vector, count, offset, flags, iobref, xdata);
+        return 0;
+}
+
+int
+trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_STATFS].enabled) {
+                char  string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique, (loc->inode)?
+                          uuid_utoa (loc->inode->gfid):"0", loc->path);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_statfs_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->statfs,
+                    loc, xdata);
+        return 0;
+}
+
+int
+trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FLUSH].enabled) {
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_flush_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->flush,
+                    fd, xdata);
+        return 0;
+}
+
+int
+trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), flags, fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_fsync_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsync,
+                    fd, flags, xdata);
+        return 0;
+}
+
+int
+trace_setxattr (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_setxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+        return 0;
+}
+
+int
+trace_getxattr (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, const char *name, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_getxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+int
+trace_removexattr (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, const char *name, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_removexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
+
+        return 0;
+}
+
+int
+trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path, fd);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_opendir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+        return 0;
+}
+
+int
+trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, dict_t *dict)
+{
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READDIRP].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64" dict=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, dict);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_readdirp_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdirp,
+                    fd, size, offset, dict);
+
+        return 0;
+}
+
+int
+trace_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               size_t size, off_t offset, dict_t *xdata)
 {
-	ERR_EINVAL_NORETURN (!this || !loc);
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READDIR].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64,
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
 
-	if (trace_fop_names[GF_FOP_INODELK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, cmd=%s)",
-			frame->root->unique, loc->path, loc->inode->ino, 
-			((cmd == F_SETLK)? "F_SETLK" : "unknown"));
-	}
+out:
+        STACK_WIND (frame, trace_readdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir,
+                    fd, size, offset, xdata);
 
-	STACK_WIND (frame, 
-		    trace_inodelk_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->inodelk,
-		    loc, cmd, flock);
-	return 0;
+        return 0;
 }
 
+int
+trace_fsyncdir (call_frame_t *frame, xlator_t *this,
+                fd_t *fd, int32_t datasync, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s datasync=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), datasync, fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
 
-int32_t 
-trace_finodelk_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
+out:
+        STACK_WIND (frame, trace_fsyncdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsyncdir,
+                    fd, datasync, xdata);
+        return 0;
+}
+
+int
+trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+              dict_t *xdata)
 {
-	ERR_EINVAL_NORETURN (!this );
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_ACCESS].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          loc->path, mask);
+
+                frame->local = loc->inode->gfid;
 
-	if (trace_fop_names[GF_FOP_FINODELK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL,
-			"%"PRId64": op_ret=%d, op_errno=%d",
-			frame->root->unique, op_ret, op_errno);
-	}
+                LOG_ELEMENT (conf, string);
+        }
 
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+out:
+        STACK_WIND (frame, trace_access_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->access,
+                    loc, mask, xdata);
+        return 0;
 }
 
 int32_t
-trace_finodelk (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd, int32_t cmd, struct flock *flock)
+trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 int32_t len, dict_t *xdata)
 {
-	ERR_EINVAL_NORETURN (!this || !fd);
 
-	if (trace_fop_names[GF_FOP_FINODELK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (fd=%p, cmd=%s)",
-			frame->root->unique, fd, 
-			((cmd == F_SETLK) ? "F_SETLK" : "unknown"));
-	}
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64
+                          "len=%u fd=%p", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, len, fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_rchecksum_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rchecksum,
+                    fd, offset, len, xdata);
+
+        return 0;
 
-	STACK_WIND (frame, 
-		    trace_finodelk_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->finodelk,
-		    fd, cmd, flock);
-	return 0;
 }
 
+int32_t
+trace_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+                fd_t *fd, const char *basename, entrylk_cmd cmd,
+                entrylk_type type, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+                char      string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd=%p "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_fentrylk_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fentrylk,
+                    volume, fd, basename, cmd, type, xdata);
+        return 0;
+
+}
 
 int32_t
-trace_xattrop (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       gf_xattrop_flags_t flags,
-	       dict_t *dict)
+trace_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
 {
-	ERR_EINVAL_NORETURN (!this || !loc);
+        trace_conf_t   *conf = NULL;
 
-	if (trace_fop_names[GF_FOP_XATTROP].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (path=%s, ino=%"PRIu64" flags=%d)",
-			frame->root->unique, loc->path, loc->inode->ino, flags);
-			
-	}
-  
-	STACK_WIND (frame, trace_xattrop_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->xattrop, 
-		    loc, flags, dict);
+        conf = this->private;
 
-	return 0;
+        if (!conf->log_file && !conf->log_history)
+                goto out;
+        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p name=%s",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, name);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_fgetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr,
+                    fd, name, xdata);
+        return 0;
 }
 
 int32_t
-trace_fxattrop (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		gf_xattrop_flags_t flags,
-		dict_t *dict)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);
-
-	if (trace_fop_names[GF_FOP_FXATTROP].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (fd=%p, flags=%d)",
-			frame->root->unique, fd, flags);
-			
-	}
-  
-	STACK_WIND (frame, trace_fxattrop_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->fxattrop, 
-		    fd, flags, dict);
-
-	return 0;
-}
-
-int32_t 
-trace_lookup (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      dict_t *xattr_req)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_LOOKUP].enabled) {  
-		/* TODO: print all the keys mentioned in xattr_req */
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"})",
-			frame->root->unique, loc->path,
-			loc->inode->ino);
-	}
-  
-	STACK_WIND (frame, trace_lookup_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->lookup, 
-		    loc, xattr_req);
-
-	return 0;
-}
-
-int32_t 
-trace_stat (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	ERR_EINVAL_NORETURN (!this || !loc );
-
-
-	if (trace_fop_names[GF_FOP_STAT].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"})",
-			frame->root->unique, loc->path, loc->inode->ino);
-	}
-
-	STACK_WIND (frame, 
-		    trace_stat_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->stat, 
-		    loc);
-  
-	return 0;
-}
-
-int32_t 
-trace_readlink (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		size_t size)
-{
-	ERR_EINVAL_NORETURN (!this || !loc || (size < 1));
-
-	if (trace_fop_names[GF_FOP_READLINK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, size=%"GF_PRI_SIZET")",
-			frame->root->unique, loc->path, loc->inode->ino, size);
-	}
-
-	STACK_WIND (frame, 
-		    trace_readlink_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->readlink, 
-		    loc, 
-		    size);
-  
-	return 0;
-}
-
-int32_t 
-trace_mknod (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode,
-	     dev_t dev)
-{
-	ERR_EINVAL_NORETURN (!this || !loc->path);
-
-	if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, mode=%d, dev=%"GF_PRI_DEV")",
-			frame->root->unique, loc->path, loc->inode->ino, mode, dev);
-	}
-
-	STACK_WIND (frame, 
-		    trace_mknod_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->mknod, 
-		    loc,
-		    mode, 
-		    dev);
-  
-	return 0;
-}
-
-int32_t 
-trace_mkdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode)
-{
-	ERR_EINVAL_NORETURN (!this || !loc || !loc->path);
-
-	if (trace_fop_names[GF_FOP_MKDIR].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (path=%s, ino=%"PRIu64", mode=%d)",
-			frame->root->unique, loc->path, 
-			((loc->inode)? loc->inode->ino : 0), mode);
-	}
-  
-	STACK_WIND (frame, 
-		    trace_mkdir_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->mkdir, 
-		    loc,
-		    mode);
-	return 0;
-}
-
-int32_t 
-trace_unlink (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_UNLINK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"})",
-			frame->root->unique, loc->path, loc->inode->ino);
-	}
-
-	STACK_WIND (frame, 
-		    trace_unlink_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->unlink, 
-		    loc);
-	return 0;
-}
-
-int32_t 
-trace_rmdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_RMDIR].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"})",
-			frame->root->unique, loc->path, loc->inode->ino);
-	}
-
-	STACK_WIND (frame, 
-		    trace_rmdir_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->rmdir, 
-		    loc);
-  
-	return 0;
-}
-
-int32_t 
-trace_symlink (call_frame_t *frame,
-	       xlator_t *this,
-	       const char *linkpath,
-	       loc_t *loc)
-{
-	ERR_EINVAL_NORETURN (!this || !linkpath || !loc || !loc->path);
-
-	if (trace_fop_names[GF_FOP_SYMLINK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (linkpath=%s, loc {path=%s, ino=%"PRIu64"})",
-			frame->root->unique, linkpath, loc->path, 
-			((loc->inode)? loc->inode->ino : 0));
-	}
-
-	STACK_WIND (frame, 
-		    trace_symlink_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->symlink, 
-		    linkpath,
-		    loc);
-  
-	return 0;
-}
-
-int32_t 
-trace_rename (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *oldloc,
-	      loc_t *newloc)
-{  
-	ERR_EINVAL_NORETURN (!this || !oldloc || !newloc);
-
-	if (trace_fop_names[GF_FOP_RENAME].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (oldloc {path=%s, ino=%"PRIu64"}, "
-			"newloc{path=%s, ino=%"PRIu64"})",
-			frame->root->unique, oldloc->path, oldloc->ino, 
-			newloc->path, newloc->ino);
-	}
-
-	STACK_WIND (frame, 
-		    trace_rename_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->rename, 
-		    oldloc,
-		    newloc);
-  
-	return 0;
-}
-
-int32_t 
-trace_link (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *oldloc,
-	    loc_t *newloc)
-{
-	ERR_EINVAL_NORETURN (!this || !oldloc || !newloc);
-
-	if (trace_fop_names[GF_FOP_LINK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (oldloc {path=%s, ino=%"PRIu64"}, "
-			"newloc {path=%s, ino=%"PRIu64"})",
-			frame->root->unique, oldloc->path, oldloc->inode->ino, 
-			newloc->path, newloc->inode->ino);
-	}
-
-	STACK_WIND (frame, 
-		    trace_link_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->link, 
-		    oldloc,
-		    newloc);
-	return 0;
-}
-
-int32_t 
-trace_chmod (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_CHMOD].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, mode=%o)",
-			frame->root->unique, loc->path, loc->inode->ino, mode);
-	}
-
-	STACK_WIND (frame, 
-		    trace_chmod_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->chmod, 
-		    loc,
-		    mode);
-  
-	return 0;
-}
-
-int32_t 
-trace_chown (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     uid_t uid,
-	     gid_t gid)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_CHOWN].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, uid=%d, gid=%d)",
-			frame->root->unique, loc->path, loc->inode->ino, uid, gid);
-	}
-
-	STACK_WIND (frame, 
-		    trace_chown_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->chown, 
-		    loc,
-		    uid,
-		    gid);
-
-	return 0;
-}
-
-int32_t 
-trace_truncate (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		off_t offset)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_TRUNCATE].enabled) { 
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, offset=%"PRId64")",
-			frame->root->unique, loc->path, loc->inode->ino, offset);
-	}
-
-	STACK_WIND (frame, 
-		    trace_truncate_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->truncate, 
-		    loc,
-		    offset);
-  
-	return 0;
-}
-
-int32_t 
-trace_utimens (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       struct timespec tv[2])
-{
-	char actime_str[256];
-	char modtime_str[256];
-  
-	ERR_EINVAL_NORETURN (!this || !loc || !tv);
-
-	if (trace_fop_names[GF_FOP_UTIMENS].enabled) {  
-		strftime (actime_str, 256, "[%b %d %H:%M:%S]", localtime (&tv[0].tv_sec));
-		strftime (modtime_str, 256, "[%b %d %H:%M:%S]", localtime (&tv[1].tv_sec));
-
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, "
-			"*tv=%p {actime=%s, modtime=%s})",
-			frame->root->unique, loc->path, loc->inode->ino, 
-			tv, actime_str, modtime_str);
-	}
-
-	STACK_WIND (frame, 
-		    trace_utimens_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->utimens, 
-		    loc,
-		    tv);
-
-	return 0;
-}
-
-int32_t 
-trace_open (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t flags,
-	    fd_t *fd)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_OPEN].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, flags=%d, fd=%p)",
-			frame->root->unique, loc->path, loc->inode->ino, flags, fd);
-	}
-
-	STACK_WIND (frame, 
-		    trace_open_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->open, 
-		    loc,
-		    flags,
-		    fd);
-	return 0;
-}
-
-int32_t 
-trace_create (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flags,
-	      mode_t mode,
-	      fd_t *fd)
-{
-	ERR_EINVAL_NORETURN (!this || !loc->path);
-
-	if (trace_fop_names[GF_FOP_CREATE].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, flags=0%o mode=0%o)",
-			frame->root->unique, loc->path, loc->inode->ino, flags, mode);
-	}
-
-	STACK_WIND (frame, 
-		    trace_create_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->create, 
-		    loc,
-		    flags,
-		    mode,
-		    fd);
-	return 0;
-}
-
-int32_t 
-trace_readv (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     size_t size,
-	     off_t offset)
-{
-	ERR_EINVAL_NORETURN (!this || !fd || (size < 1));
-
-	if (trace_fop_names[GF_FOP_READ].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (*fd=%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-			frame->root->unique, fd, size, offset);
-	}
-
-	STACK_WIND (frame, 
-		    trace_readv_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->readv,
-		    fd,
-		    size,
-		    offset);
-	return 0;
-}
-
-int32_t 
-trace_writev (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      struct iovec *vector,
-	      int32_t count,
-	      off_t offset)
-{
-	ERR_EINVAL_NORETURN (!this || !fd || !vector || (count < 1));
-
-	if (trace_fop_names[GF_FOP_WRITE].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (*fd=%p, *vector=%p, count=%d, offset=%"PRId64")",
-			frame->root->unique, fd, vector, count, offset);
-	}
-
-	STACK_WIND (frame, 
-		    trace_writev_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->writev, 
-		    fd,
-		    vector,
-		    count,
-		    offset);
-	return 0;
-}
-
-int32_t 
-trace_statfs (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_STATFS].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"})",
-			frame->root->unique, loc->path, 
-			((loc->inode)? loc->inode->ino : 0));
-	}
-
-	STACK_WIND (frame, 
-		    trace_statfs_cbk, 
-		    FIRST_CHILD(this), FIRST_CHILD(this)->fops->statfs, 
-		    loc);
-	return 0; 
-}
-
-int32_t 
-trace_flush (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);
-
-	if (trace_fop_names[GF_FOP_FLUSH].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (*fd=%p)", 
-			frame->root->unique, fd);
-	}
-
-	STACK_WIND (frame, 
-		    trace_flush_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->flush, 
-		    fd);
-	return 0;
-}
-
-
-int32_t 
-trace_fsync (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);
-
-	if (trace_fop_names[GF_FOP_FSYNC].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (flags=%d, *fd=%p)", 
-			frame->root->unique, flags, fd);
-	}
-
-	STACK_WIND (frame, 
-		    trace_fsync_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->fsync, 
-		    fd,
-		    flags);
-	return 0;
-}
-
-int32_t 
-trace_setxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		dict_t *dict,
-		int32_t flags)
-{
-	ERR_EINVAL_NORETURN (!this || !loc || !dict);
-
-	if (trace_fop_names[GF_FOP_SETXATTR].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, dict=%p, flags=%d)",
-			frame->root->unique, loc->path, 
-			((loc->inode)? loc->inode->ino : 0), dict, flags);
-	}
-
-	STACK_WIND (frame, 
-		    trace_setxattr_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->setxattr, 
-		    loc,
-		    dict,
-		    flags);
-	return 0;
-}
-
-int32_t 
-trace_getxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		const char *name)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_GETXATTR].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}), name=%s",
-			frame->root->unique, loc->path, 
-			((loc->inode)? loc->inode->ino : 0), name);
-	}
-
-	STACK_WIND (frame, 
-		    trace_getxattr_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc,
-		    name);
-	return 0;
-}
-
-int32_t 
-trace_removexattr (call_frame_t *frame,
-		   xlator_t *this,
-		   loc_t *loc,
-		   const char *name)
-{
-	ERR_EINVAL_NORETURN (!this || !loc || !name);
-
-	if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (loc {path=%s, ino=%"PRIu64"}, name=%s)",
-			frame->root->unique, loc->path, 
-			((loc->inode)? loc->inode->ino : 0), name);
-	}
-
-	STACK_WIND (frame, 
-		    trace_removexattr_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->removexattr, 
-		    loc,
-		    name);
-
-	return 0;
-}
-
-int32_t 
-trace_opendir (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       fd_t *fd)
-{
-	ERR_EINVAL_NORETURN (!this || !loc );
-
-	if (trace_fop_names[GF_FOP_OPENDIR].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64":( loc {path=%s, ino=%"PRIu64"}, fd=%p)",
-			frame->root->unique, loc->path, loc->inode->ino, fd);
-	}
-
-	STACK_WIND (frame, 
-		    trace_opendir_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->opendir, 
-		    loc,
-		    fd);
-	return 0;
-}
-
-int32_t 
-trace_getdents (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		size_t size,
-		off_t offset,
-		int32_t flag)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);  
-
-	if (trace_fop_names[GF_FOP_GETDENTS].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (fd=%p, size=%"GF_PRI_SIZET", offset=%"PRId64", flag=0x%x)",
-			frame->root->unique, fd, size, offset, flag);
-	}
-
-	STACK_WIND (frame, 
-		    trace_getdents_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->getdents, 
-		    fd,
-		    size, 
-		    offset, 
-		    flag);
-	return 0;
-}
-
-
-int32_t 
-trace_readdir (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       size_t size,
-	       off_t offset)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);  
-
-	if (trace_fop_names[GF_FOP_READDIR].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (fd=%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-			frame->root->unique, fd, size, offset);
-	}
-
-	STACK_WIND (frame, 
-		    trace_readdir_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->readdir,
-		    fd,
-		    size, 
-		    offset);
-
-	return 0;
-}
-
-
-int32_t 
-trace_fsyncdir (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		int32_t datasync)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);
-
-	if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (datasync=%d, *fd=%p)", 
-			frame->root->unique, datasync, fd);
-	}
-
-	STACK_WIND (frame, 
-		    trace_fsyncdir_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->fsyncdir, 
-		    fd,
-		    datasync);
-	return 0;
-}
-
-int32_t 
-trace_access (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t mask)
-{
-	ERR_EINVAL_NORETURN (!this || !loc);
-
-	if (trace_fop_names[GF_FOP_ACCESS].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (*loc {path=%s, ino=%"PRIu64"}, mask=0%o)",
-			frame->root->unique, loc->path, 
-			((loc->inode)? loc->inode->ino : 0), mask);
-	}
-
-	STACK_WIND (frame, 
-		    trace_access_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->access, 
-		    loc,
-		    mask);
-	return 0;
-}
-
-int32_t 
-trace_ftruncate (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 off_t offset)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);
-
-	if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (offset=%"PRId64", *fd=%p)", 
-			frame->root->unique, offset, fd);
-	}
-
-	STACK_WIND (frame, 
-		    trace_ftruncate_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->ftruncate, 
-		    fd,
-		    offset);
-
-	return 0;
-}
-
-int32_t 
-trace_fchown (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      uid_t uid,
-	      gid_t gid)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);
-
-	if (trace_fop_names[GF_FOP_FCHOWN].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (*fd=%p, uid=%d, gid=%d)", 
-			frame->root->unique, fd, uid, gid);
-	}
-
-	STACK_WIND (frame, 
-		    trace_fchown_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->fchown, 
-		    fd,
-		    uid,
-		    gid);
-	return 0;
-}
-
-int32_t 
-trace_fchmod (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      mode_t mode)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);
-
-	if (trace_fop_names[GF_FOP_FCHMOD].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (mode=%o, *fd=%p)", 
-			frame->root->unique, mode, fd);
-	}
-
-	STACK_WIND (frame, 
-		    trace_fchmod_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->fchmod, 
-		    fd,
-		    mode);
-	return 0;
-}
-
-int32_t 
-trace_fstat (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);
-
-	if (trace_fop_names[GF_FOP_FSTAT].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (*fd=%p)", 
-			frame->root->unique, fd);
-	}
-
-	STACK_WIND (frame, 
-		    trace_fstat_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->fstat, 
-		    fd);
-	return 0;
-}
-
-int32_t 
-trace_lk (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  int32_t cmd,
-	  struct flock *lock)
-{
-	ERR_EINVAL_NORETURN (!this || !fd);
-
-	if (trace_fop_names[GF_FOP_LK].enabled) {  
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (*fd=%p, cmd=%d, lock {l_type=%d, l_whence=%d, "
-			"l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-			frame->root->unique, fd, cmd, lock->l_type, lock->l_whence, 
-			lock->l_start, lock->l_len, lock->l_pid);
-	}
-
-	STACK_WIND (frame, 
-		    trace_lk_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->lk, 
-		    fd,
-		    cmd,
-		    lock);
-	return 0;
-}
-
-int32_t 
-trace_setdents (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		int32_t flags,
-		dir_entry_t *entries,
-		int32_t count)
-{
-	if (trace_fop_names[GF_FOP_SETDENTS].enabled) {
-		gf_log (this->name, GF_LOG_NORMAL, 
-			"%"PRId64": (*fd=%p, flags=%d, count=%d", 
-			frame->root->unique, fd, flags, count);
-	}
-
-	STACK_WIND (frame, 
-		    trace_setdents_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->setdents, 
-		    fd,
-		    flags,
-		    entries,
-		    count);
-	return 0;
+trace_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_fsetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
 }
 
+int
+trace_ftruncate (call_frame_t *frame, xlator_t *this,
+                 fd_t *fd, off_t offset, dict_t *xdata)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_ftruncate_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate,
+                    fd, offset, xdata);
+
+        return 0;
+}
+
+int
+trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_fstat_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+        return 0;
+}
+
+int
+trace_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
+          int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LK].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, cmd=%d, "
+                          "lock {l_type=%d, "
+                          "l_whence=%d, l_start=%"PRId64", "
+                          "l_len=%"PRId64", l_pid=%u})",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, cmd,
+                          lock->l_type, lock->l_whence,
+                          lock->l_start, lock->l_len, lock->l_pid);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_lk_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lk,
+                    fd, cmd, lock, xdata);
+        return 0;
+}
 
 int32_t
-trace_checksum_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    uint8_t *fchecksum,
-		    uint8_t *dchecksum)
+trace_forget (xlator_t *this, inode_t *inode)
 {
-	gf_log (this->name, GF_LOG_NORMAL, 
-		"%"PRId64": op_ret (%d), op_errno(%d)", 
-		frame->root->unique, op_ret, op_errno);
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+        /* If user want to understand when a lookup happens,
+           he should know about 'forget' too */
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s", uuid_utoa (inode->gfid));
 
-	STACK_UNWIND (frame, op_ret, op_errno, fchecksum, dchecksum);
+                LOG_ELEMENT (conf, string);
+        }
 
-	return 0;
+out:
+        return 0;
 }
 
 int32_t
-trace_checksum (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		int32_t flag)
-{
-	gf_log (this->name, GF_LOG_NORMAL, 
-		"%"PRId64": loc->path (%s) flag (%d)", 
-		frame->root->unique, loc->path, flag);
-  
-	STACK_WIND (frame,
-		    trace_checksum_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->checksum, 
-		    loc,
-		    flag);
-
-	return 0;
-}
-
-
-int32_t 
-trace_stats_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct xlator_stats *stats)
-{
-	gf_log (this->name, GF_LOG_NORMAL, 
-		"%"PRId64": op_ret (%d), op_errno(%d)", 
-		frame->root->unique, op_ret, op_errno);
-
-	STACK_UNWIND (frame, op_ret, op_errno, stats);
-	return 0;
-}
-
-int32_t 
-trace_stats (call_frame_t *frame,
-	     xlator_t *this, 
-	     int32_t flags)
-{
-	ERR_EINVAL_NORETURN (!this);
-  
-	gf_log (this->name, GF_LOG_NORMAL, 
-		"%"PRId64": (flags=%d)", 
-		frame->root->unique, flags);
-	
-	STACK_WIND (frame, 
-		    trace_stats_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->mops->stats, 
-		    flags);
-
-	return 0;
+trace_releasedir (xlator_t *this, fd_t *fd)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        return 0;
 }
 
+int32_t
+trace_release (xlator_t *this, fd_t *fd)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_OPEN].enabled ||
+            trace_fop_names[GF_FOP_CREATE].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        return 0;
+}
+
+
 void
 enable_all_calls (int enabled)
 {
-	int i;
-	for (i = 0; i < GF_FOP_MAXVALUE; i++)
-		trace_fop_names[i].enabled = enabled;
+        int i;
+
+        for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                trace_fop_names[i].enabled = enabled;
 }
 
-void 
+void
 enable_call (const char *name, int enabled)
 {
-	int i;
-	for (i = 0; i < GF_FOP_MAXVALUE; i++)
-		if (!strcasecmp(trace_fop_names[i].name, name))
-			trace_fop_names[i].enabled = enabled;
+        int i;
+        for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                if (!strcasecmp(trace_fop_names[i].name, name))
+                        trace_fop_names[i].enabled = enabled;
 }
 
 
-/* 
-   include = 1 for "include-ops"
-           = 0 for "exclude-ops" 
+/*
+  include = 1 for "include-ops"
+  = 0 for "exclude-ops"
 */
 void
 process_call_list (const char *list, int include)
 {
-	enable_all_calls (include ? 0 : 1);
-	
-	char *call = strsep ((char **)&list, ",");
-	while (call) {
-		enable_call (call, include);
-		call = strsep ((char **)&list, ",");
-	}
+        enable_all_calls (include ? 0 : 1);
+
+        char *call = strsep ((char **)&list, ",");
+
+        while (call) {
+                enable_call (call, include);
+                call = strsep ((char **)&list, ",");
+        }
+}
+
+int32_t
+trace_dump_history (xlator_t *this)
+{
+        int ret = -1;
+        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
+        trace_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("trace", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
+
+        conf = this->private;
+        // Is it ok to return silently if log-history option his off?
+        if (conf && conf->log_history == _gf_true) {
+                gf_proc_dump_build_key (key_prefix, "xlator.debug.trace",
+                                        "history");
+                gf_proc_dump_add_section (key_prefix);
+                eh_dump (this->history, NULL, dump_history_trace);
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_trace_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
 }
 
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int32_t       ret            = -1;
+        trace_conf_t    *conf           = NULL;
+        char            *includes = NULL, *excludes = NULL;
+
+        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+        GF_VALIDATE_OR_GOTO (this->name, options, out);
+
+        conf = this->private;
+
+        includes = data_to_str (dict_get (options, "include-ops"));
+        excludes = data_to_str (dict_get (options, "exclude-ops"));
+
+        {
+                int i;
+                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
+                        trace_fop_names[i].enabled = 1;
+                }
+        }
+
+        if (includes && excludes) {
+                gf_log (this->name,
+                        GF_LOG_ERROR,
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
+                goto out;
+        }
+
+        if (includes)
+                process_call_list (includes, 1);
+        if (excludes)
+                process_call_list (excludes, 0);
+
+        /* Should resizing of the event-history be allowed in reconfigure?
+         * for which a new event_history might have to be allocated and the
+         * older history has to be freed.
+         */
+        GF_OPTION_RECONF ("log-file", conf->log_file, options, bool, out);
+
+        GF_OPTION_RECONF ("log-history", conf->log_history, options, bool, out);
 
-int32_t 
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int32_t
 init (xlator_t *this)
 {
-  dict_t *options = this->options;
-  char *includes = NULL, *excludes = NULL;
-  
-  if (!this)
-	  return -1;
-  
-  if (!this->children || this->children->next) {
-	  gf_log (this->name, GF_LOG_ERROR, 
-		  "trace translator requires one subvolume");
-	  return -1;
-  }
-  if (!this->parents) {
-	  gf_log (this->name, GF_LOG_WARNING,
-		  "dangling volume. check volfile ");
-  }
-  
-  
-  includes = data_to_str (dict_get (options, "include-ops"));
-  excludes = data_to_str (dict_get (options, "exclude-ops"));
-  
-  {
-	  int i;
-	  for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-		  trace_fop_names[i].name = (gf_fop_list[i] ?
-					     gf_fop_list[i] : ":O");
-		  trace_fop_names[i].enabled = 1;
-	  }
-  }
-  
-  if (includes && excludes) {
-	  gf_log (this->name, 
-		  GF_LOG_ERROR,
-		  "must specify only one of 'include-ops' and 'exclude-ops'");
-	  return -1;
-  }
-  if (includes)
-	  process_call_list (includes, 1);
-  if (excludes)
-	  process_call_list (excludes, 0);
-  
-  gf_log_set_loglevel (GF_LOG_NORMAL);
-  
-  /* Set this translator's inode table pointer to child node's pointer. */
-  this->itable = FIRST_CHILD (this)->itable;
-  
-  return 0;
+        dict_t *options = NULL;
+        char *includes = NULL, *excludes = NULL;
+        char *forced_loglevel = NULL;
+        eh_t *history = NULL;
+        int  ret = -1;
+        size_t  history_size = TRACE_DEFAULT_HISTORY_SIZE;
+        trace_conf_t    *conf = NULL;
+
+        if (!this)
+                return -1;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "trace translator requires one subvolume");
+                return -1;
+        }
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+        conf = GF_CALLOC (1, sizeof (trace_conf_t), gf_trace_mt_trace_conf_t);
+        if (!conf) {
+                gf_log (this->name, GF_LOG_ERROR, "cannot allocate "
+                        "xl->private");
+                return -1;
+        }
+
+        options = this->options;
+        includes = data_to_str (dict_get (options, "include-ops"));
+        excludes = data_to_str (dict_get (options, "exclude-ops"));
+
+        {
+                int i;
+                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
+                        trace_fop_names[i].enabled = 1;
+                }
+        }
+
+        if (includes && excludes) {
+                gf_log (this->name,
+                        GF_LOG_ERROR,
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
+                return -1;
+        }
+
+        if (includes)
+                process_call_list (includes, 1);
+        if (excludes)
+                process_call_list (excludes, 0);
+
+
+        GF_OPTION_INIT ("history-size", conf->history_size, size, out);
+
+        gf_log (this->name, GF_LOG_INFO, "history size %"GF_PRI_SIZET,
+                history_size);
+
+        GF_OPTION_INIT ("log-file", conf->log_file, bool, out);
+
+        gf_log (this->name, GF_LOG_INFO, "logging to file %s",
+                (conf->log_file == _gf_true)?"enabled":"disabled");
+
+        GF_OPTION_INIT ("log-history", conf->log_history, bool, out);
+
+        gf_log (this->name, GF_LOG_DEBUG, "logging to history %s",
+                (conf->log_history == _gf_true)?"enabled":"disabled");
+
+        history = eh_new (history_size, _gf_false, NULL);
+        if (!history) {
+                gf_log (this->name, GF_LOG_ERROR, "event history cannot be "
+                        "initialized");
+                return -1;
+        }
+
+        this->history = history;
+
+        conf->trace_log_level = GF_LOG_INFO;
+
+        if (dict_get (options, "force-log-level")) {
+                forced_loglevel = data_to_str (dict_get (options,
+                                                         "force-log-level"));
+                if (!forced_loglevel)
+                        goto setloglevel;
+
+                if (strcmp (forced_loglevel, "INFO") == 0)
+                        conf->trace_log_level = GF_LOG_INFO;
+                else if (strcmp (forced_loglevel, "TRACE") == 0)
+                        conf->trace_log_level = GF_LOG_TRACE;
+                else if (strcmp (forced_loglevel, "ERROR") == 0)
+                        conf->trace_log_level = GF_LOG_ERROR;
+                else if (strcmp (forced_loglevel, "DEBUG") == 0)
+                        conf->trace_log_level = GF_LOG_DEBUG;
+                else if (strcmp (forced_loglevel, "WARNING") == 0)
+                        conf->trace_log_level = GF_LOG_WARNING;
+                else if (strcmp (forced_loglevel, "CRITICAL") == 0)
+                        conf->trace_log_level = GF_LOG_CRITICAL;
+                else if (strcmp (forced_loglevel, "NONE") == 0)
+                        conf->trace_log_level = GF_LOG_NONE;
+        }
+
+setloglevel:
+        gf_log_set_loglevel (conf->trace_log_level);
+        this->private = conf;
+        ret = 0;
+out:
+        if (ret == -1) {
+                if (history)
+                        GF_FREE (history);
+                if (conf)
+                        GF_FREE (conf);
+        }
+
+        return ret;
 }
 
 void
 fini (xlator_t *this)
 {
-	if (!this)
-		return;
-	
-	gf_log (this->name, GF_LOG_NORMAL, 
-		"trace translator unloaded");
-	return;
+        if (!this)
+                return;
+
+        if (this->history)
+                eh_destroy (this->history);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "trace translator unloaded");
+        return;
 }
 
 struct xlator_fops fops = {
-  .stat        = trace_stat,
-  .readlink    = trace_readlink,
-  .mknod       = trace_mknod,
-  .mkdir       = trace_mkdir,
-  .unlink      = trace_unlink,
-  .rmdir       = trace_rmdir,
-  .symlink     = trace_symlink,
-  .rename      = trace_rename,
-  .link        = trace_link,
-  .chmod       = trace_chmod,
-  .chown       = trace_chown,
-  .truncate    = trace_truncate,
-  .utimens     = trace_utimens,
-  .open        = trace_open,
-  .readv       = trace_readv,
-  .writev      = trace_writev,
-  .statfs      = trace_statfs,
-  .flush       = trace_flush,
-  .fsync       = trace_fsync,
-  .setxattr    = trace_setxattr,
-  .getxattr    = trace_getxattr,
-  .removexattr = trace_removexattr,
-  .opendir     = trace_opendir,
-  .readdir     = trace_readdir, 
-  .fsyncdir    = trace_fsyncdir,
-  .access      = trace_access,
-  .ftruncate   = trace_ftruncate,
-  .fstat       = trace_fstat,
-  .create      = trace_create,
-  .fchown      = trace_fchown,
-  .fchmod      = trace_fchmod,
-  .lk          = trace_lk,
-  .inodelk     = trace_inodelk,
-  .finodelk    = trace_finodelk,
-  .entrylk     = trace_entrylk,
-  .lookup      = trace_lookup,
-  .setdents    = trace_setdents,
-  .getdents    = trace_getdents,
-  .checksum    = trace_checksum,
-  .xattrop     = trace_xattrop,
-  .fxattrop    = trace_fxattrop,
-};
-
-struct xlator_mops mops = {
-	.stats    = trace_stats,
+        .stat        = trace_stat,
+        .readlink    = trace_readlink,
+        .mknod       = trace_mknod,
+        .mkdir       = trace_mkdir,
+        .unlink      = trace_unlink,
+        .rmdir       = trace_rmdir,
+        .symlink     = trace_symlink,
+        .rename      = trace_rename,
+        .link        = trace_link,
+        .truncate    = trace_truncate,
+        .open        = trace_open,
+        .readv       = trace_readv,
+        .writev      = trace_writev,
+        .statfs      = trace_statfs,
+        .flush       = trace_flush,
+        .fsync       = trace_fsync,
+        .setxattr    = trace_setxattr,
+        .getxattr    = trace_getxattr,
+        .fsetxattr   = trace_fsetxattr,
+        .fgetxattr   = trace_fgetxattr,
+        .removexattr = trace_removexattr,
+        .opendir     = trace_opendir,
+        .readdir     = trace_readdir,
+        .readdirp    = trace_readdirp,
+        .fsyncdir    = trace_fsyncdir,
+        .access      = trace_access,
+        .ftruncate   = trace_ftruncate,
+        .fstat       = trace_fstat,
+        .create      = trace_create,
+        .lk          = trace_lk,
+        .inodelk     = trace_inodelk,
+        .finodelk    = trace_finodelk,
+        .entrylk     = trace_entrylk,
+        .fentrylk    = trace_fentrylk,
+        .lookup      = trace_lookup,
+        .rchecksum   = trace_rchecksum,
+        .xattrop     = trace_xattrop,
+        .fxattrop    = trace_fxattrop,
+        .setattr     = trace_setattr,
+        .fsetattr    = trace_fsetattr,
 };
 
 struct xlator_cbks cbks = {
+        .release     = trace_release,
+        .releasedir  = trace_releasedir,
+        .forget      = trace_forget,
 };
 
 struct volume_options options[] = {
-	{ .key  = {"include-ops", "include"}, 
-	  .type = GF_OPTION_TYPE_STR,
-	  /*.value = { ""} */
-	},
-	{ .key  = {"exclude-ops", "exclude"}, 
-	  .type = GF_OPTION_TYPE_STR 
-	  /*.value = { ""} */	  
-	},
-	{ .key  = {NULL} },
+        { .key  = {"include-ops", "include"},
+          .type = GF_OPTION_TYPE_STR,
+          /*.value = { ""} */
+        },
+        { .key  = {"exclude-ops", "exclude"},
+          .type = GF_OPTION_TYPE_STR
+          /*.value = { ""} */
+        },
+        { .key  = {"history-size"},
+          .type = GF_OPTION_TYPE_SIZET,
+          .default_value = "1024",
+        },
+        { .key  = {"log-file"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
+        { .key  = {"log-history"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
+        { .key  = {NULL} },
 };
 
+struct xlator_dumpops dumpops = {
+        .history = trace_dump_history
+};
diff --git a/xlators/debug/trace/src/trace.h b/xlators/debug/trace/src/trace.h
new file mode 100644
index 000000000..045eefb36
--- /dev/null
+++ b/xlators/debug/trace/src/trace.h
@@ -0,0 +1,98 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <time.h>
+#include <errno.h>
+#include "glusterfs.h"
+#include "xlator.h"
+#include "common-utils.h"
+#include "event-history.h"
+#include "logging.h"
+#include "circ-buff.h"
+#include "statedump.h"
+#include "options.h"
+
+#define TRACE_DEFAULT_HISTORY_SIZE 1024
+
+typedef struct {
+        /* Since the longest fop name is fremovexattr i.e 12 characters, array size
+         * is kept 24, i.e double of the maximum.
+         */
+        char name[24];
+        int enabled;
+} trace_fop_name_t;
+
+trace_fop_name_t trace_fop_names[GF_FOP_MAXVALUE];
+
+typedef struct {
+        gf_boolean_t log_file;
+        gf_boolean_t log_history;
+        size_t       history_size;
+        int trace_log_level;
+} trace_conf_t;
+
+#define TRACE_STACK_UNWIND(op, frame, params ...)                       \
+        do {                                                            \
+                frame->local = NULL;                                    \
+                STACK_UNWIND_STRICT (op, frame, params);                \
+        } while (0);
+
+#define LOG_ELEMENT(_conf, _string)                                     \
+        do {                                                            \
+                if (_conf) {                                            \
+                        if ((_conf->log_history) == _gf_true)           \
+                                gf_log_eh ("%s", _string);              \
+                        if ((_conf->log_file) == _gf_true)              \
+                                gf_log (THIS->name, _conf->trace_log_level, \
+                                        "%s", _string);                 \
+                }                                                       \
+        } while (0);
+
+#define trace_stat_to_str(buf, statstr)                                 \
+        do {                                                            \
+                char     atime_buf[256]    = {0,};                      \
+                char     mtime_buf[256]    = {0,};                      \
+                char     ctime_buf[256]    = {0,};                      \
+                uint64_t ia_time           = 0;                         \
+                                                                        \
+                if (!buf)                                               \
+                        break;                                          \
+                                                                        \
+                ia_time = buf->ia_atime;                                \
+                strftime (atime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                ia_time = buf->ia_mtime;                                \
+                strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                ia_time = buf->ia_ctime;                                \
+                strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                snprintf (statstr, sizeof (statstr),                    \
+                          "gfid=%s ino=%"PRIu64", mode=%o, "            \
+                          "nlink=%"GF_PRI_NLINK", uid=%u, "             \
+                          "gid=%u, size=%"PRIu64", "                    \
+                          "blocks=%"PRIu64", atime=%s, "                \
+                          "mtime=%s, ctime=%s",                         \
+                          uuid_utoa (buf->ia_gfid), buf->ia_ino,        \
+                          st_mode_from_ia (buf->ia_prot,                \
+                                           buf->ia_type),               \
+                          buf->ia_nlink, buf->ia_uid,                   \
+                          buf->ia_gid, buf->ia_size,                    \
+                          buf->ia_blocks, atime_buf,                    \
+                          mtime_buf, ctime_buf);                        \
+        } while (0);
diff --git a/xlators/encryption/Makefile.am b/xlators/encryption/Makefile.am
index 2cbde680f..36efc6698 100644
--- a/xlators/encryption/Makefile.am
+++ b/xlators/encryption/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = rot-13
+SUBDIRS = rot-13 crypt
 
 CLEANFILES = 
diff --git a/xlators/cluster/unify/Makefile.am b/xlators/encryption/crypt/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/xlators/cluster/unify/Makefile.am
+++ b/xlators/encryption/crypt/Makefile.am
diff --git a/xlators/encryption/crypt/src/Makefile.am b/xlators/encryption/crypt/src/Makefile.am
new file mode 100644
index 000000000..faadd117f
--- /dev/null
+++ b/xlators/encryption/crypt/src/Makefile.am
@@ -0,0 +1,24 @@
+if ENABLE_CRYPT_XLATOR
+
+xlator_LTLIBRARIES = crypt.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
+
+crypt_la_LDFLAGS = -module -avoidversion -lssl -lcrypto
+
+crypt_la_SOURCES = keys.c data.c metadata.c atom.c crypt.c
+crypt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
+else
+
+noinst_DIST = keys.c data.c metadata.c atom.c crypt.c
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+endif
\ No newline at end of file
diff --git a/xlators/encryption/crypt/src/atom.c b/xlators/encryption/crypt/src/atom.c
new file mode 100644
index 000000000..1ec41495c
--- /dev/null
+++ b/xlators/encryption/crypt/src/atom.c
@@ -0,0 +1,962 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/*
+ *                              Glossary
+ *
+ *
+ * cblock            (or cipher block). A logical unit in a file.
+ *                   cblock size is defined as the number of bits
+ *                   in an input (or output) block of the block
+ *                   cipher (*). Cipher block size is a property of
+ *                   cipher algorithm. E.g. cblock size is 64 bits
+ *                   for DES, 128 bits for AES, etc.
+ *
+ * atomic cipher     A cipher algorithm, which requires some chunks of
+ * algorithm         text to be padded at left and(or) right sides before
+ *                   cipher transaform.
+ *
+ *
+ * block (atom)      Minimal chunk of file's data, which doesn't require
+ *                   padding. We'll consider logical units in a file of
+ *                   block size (atom size).
+ *
+ * cipher algorithm  Atomic cipher algorithm, which requires the last
+ * with EOF issue    incomplete cblock in a file to be padded with some
+ *                   data (usually zeros).
+ *
+ *
+ * operation, which  reading/writing from offset, which is not aligned to
+ * forms a gap at    to atom size
+ * the beginning
+ *
+ *
+ * operation, which  reading/writing count bytes starting from offset off,
+ * forms a gap at    so that off+count is not aligned to atom_size
+ * the end
+ *
+ * head block        the first atom affected by an operation, which forms
+ *                   a gap at the beginning, or(and) at the end.
+ *                   Сomment. Head block has at least one gap (either at
+ *                   the beginning, or at the end)
+ *
+ *
+ * tail block        the last atom different from head, affected by an
+ *                   operation, which forms a gap at the end.
+ *                   Сomment: Tail block has exactly one gap (at the end).
+ *
+ *
+ * partial block     head or tail block
+ *
+ *
+ * full block        block without gaps.
+ *
+ *
+ * (*) Recommendation for Block Cipher Modes of Operation
+ *     Methods and Techniques
+ *     NIST Special Publication 800-38A Edition 2001
+ */
+
+/*
+ * atom->offset_at()
+ */
+static off_t offset_at_head(struct avec_config *conf)
+{
+	return conf->aligned_offset;
+}
+
+static off_t offset_at_hole_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_hole_conf(frame));
+}
+
+static off_t offset_at_data_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_data_conf(frame));
+}
+
+
+static off_t offset_at_tail(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0) +
+		(conf->nr_full_blocks << get_atom_bits(object));
+}
+
+static off_t offset_at_hole_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_hole_conf(frame), object);
+}
+
+
+static off_t offset_at_data_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_data_conf(frame), object);
+}
+
+static off_t offset_at_full(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0);
+}
+
+static off_t offset_at_data_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_data_conf(frame), object);
+}
+
+static off_t offset_at_hole_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->io_size_nopad()
+ */
+
+static uint32_t io_size_nopad_head(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	uint32_t gap_at_beg;
+	uint32_t gap_at_end;
+
+	check_head_block(conf);
+
+	gap_at_beg = conf->off_in_head;
+
+	if (has_tail_block(conf) || has_full_blocks(conf) || conf->off_in_tail == 0 )
+		gap_at_end = 0;
+	else
+		gap_at_end = get_atom_size(object) - conf->off_in_tail;
+
+	return get_atom_size(object) - (gap_at_beg + gap_at_end);
+}
+
+static uint32_t io_size_nopad_tail(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	check_tail_block(conf);
+	return conf->off_in_tail;
+}
+
+static uint32_t io_size_nopad_full(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{	
+	check_full_block(conf);
+	return get_atom_size(object);
+}
+
+static uint32_t io_size_nopad_data_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_hole_conf(frame), object);
+}
+
+static uint32_t offset_in_head(struct avec_config *conf)
+{
+	check_cursor_head(conf);
+
+	return conf->off_in_head;
+}
+
+static uint32_t offset_in_tail(call_frame_t *frame,
+			       struct object_cipher_info *object)
+{
+	return 0;
+}
+
+static uint32_t offset_in_full(struct avec_config *conf,
+			       struct object_cipher_info *object)
+{
+	check_cursor_full(conf);
+
+	if (has_head_block(conf))
+		return (conf->cursor - 1) << get_atom_bits(object);
+	else
+		return conf->cursor << get_atom_bits(object);
+}
+
+static uint32_t offset_in_data_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_data_conf(frame));
+}
+
+static uint32_t offset_in_hole_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_hole_conf(frame));
+}
+
+static uint32_t offset_in_data_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_data_conf(frame), object);
+}
+
+static uint32_t offset_in_hole_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->rmw()
+ */
+/*
+ * Pre-conditions:
+ * @vec contains plain text of the latest
+ * version.
+ *
+ * Uptodate gaps of the @partial block with
+ * this plain text, encrypt the whole block
+ * and write the result to disk.
+ */
+static int32_t rmw_partial_block(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 struct iovec *vec,
+				 int32_t count,
+				 struct iatt *stbuf,
+				 struct iobref *iobref,
+				 struct rmw_atom *atom)
+{
+	size_t was_read = 0;
+	uint64_t file_size;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *partial = atom->get_iovec(frame, 0);
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_partial_block;
+#if DEBUG_CRYPT
+	gf_boolean_t check_last_cblock = _gf_false;
+#endif
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto exit;
+
+	file_size = local->cur_file_size;
+	was_read = op_ret;
+
+	if (atom->locality == HEAD_ATOM && conf->off_in_head) {
+		/*
+		 * head atom with a non-uptodate gap
+		 * at the beginning
+		 *
+		 * fill the gap with plain text of the
+		 * latest version. Convert a part of hole
+		 * (if any) to zeros.
+		 */
+		int32_t i;
+		int32_t copied = 0;
+		int32_t to_gap; /* amount of data needed to uptodate
+				   the gap at the beginning */
+#if 0
+		int32_t hole = 0; /* The part of the hole which
+				   * got in the head block */
+#endif /* 0 */
+		to_gap = conf->off_in_head;
+
+		if (was_read < to_gap) {
+			if (file_size >
+			    offset_at_head(conf) + was_read) {
+				/*
+				 * It is impossible to uptodate
+				 * head block: too few bytes have
+				 * been read from disk, so that
+				 * partial write is impossible.
+				 *
+				 * It could happen because of many
+				 * reasons: IO errors, (meta)data
+				 * corruption in the local file system,
+				 * etc.
+				 */
+				gf_log(this->name, GF_LOG_WARNING,
+				     "Can not uptodate a gap at the beginning");
+				local->op_ret = -1;
+				local->op_errno = EIO;
+				goto exit;
+			}
+#if 0
+			hole = to_gap - was_read;
+#endif /* 0 */
+			to_gap = was_read;
+		}
+		/*
+		 * uptodate the gap at the beginning
+		 */
+		for (i = 0; i < count && copied < to_gap; i++) {
+			int32_t to_copy;
+
+			to_copy = vec[i].iov_len;
+			if (to_copy > to_gap - copied)
+				to_copy = to_gap - copied;
+
+			memcpy(partial->iov_base, vec[i].iov_base, to_copy);
+			copied += to_copy;
+		}
+#if 0
+		/*
+		 * If possible, convert part of the
+		 * hole, which got in the head block
+		 */
+		ret = TRY_LOCK(&local->hole_lock);
+		if (!ret) {
+			if (local->hole_handled)
+				/*
+				 * already converted by
+				 * crypt_writev_cbk()
+				 */
+				UNLOCK(&local->hole_lock);
+			else {
+				/*
+				 * convert the part of the hole
+				 * which got in the head block
+				 * to zeros.
+				 *
+				 * Update the orig_offset to make
+				 * sure writev_cbk() won't care
+				 * about this part of the hole.
+				 *
+				 */
+				memset(partial->iov_base + to_gap, 0, hole);
+
+				conf->orig_offset -= hole;
+				conf->orig_size += hole;
+				UNLOCK(&local->hole_lock);
+			}
+		}
+		else /*
+		      * conversion is being performed
+		      * by crypt_writev_cbk()
+		      */
+			;
+#endif /* 0 */
+	}
+	if (atom->locality == TAIL_ATOM ||
+	    (!has_tail_block(conf) && conf->off_in_tail)) {
+		/*
+		 * tail atom, or head atom with a non-uptodate
+		 * gap at the end.
+		 *
+		 * fill the gap at the end of the block
+		 * with plain text of the latest version.
+		 * Pad the result, (if needed)
+		 */
+		int32_t i;
+		int32_t to_gap;
+		int copied;
+		off_t off_in_tail;
+		int32_t to_copy;
+
+		off_in_tail = conf->off_in_tail;
+		to_gap = conf->gap_in_tail;
+
+		if (to_gap && was_read < off_in_tail + to_gap) {
+			/*
+			 * It is impossible to uptodate
+			 * the gap at the end: too few bytes
+			 * have been read from disk, so that
+			 * partial write is impossible.
+			 *
+			 * It could happen because of many
+			 * reasons: IO errors, (meta)data
+			 * corruption in the local file system,
+			 * etc.
+			 */
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Can not uptodate a gap at the end");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * uptodate the gap at the end
+		 */
+		copied = 0;
+		to_copy = to_gap;
+		for(i = count - 1; i >= 0 && to_copy > 0; i--) {
+			uint32_t from_vec, off_in_vec;
+
+			off_in_vec = 0;
+			from_vec = vec[i].iov_len;
+			if (from_vec > to_copy) {
+				off_in_vec = from_vec - to_copy;
+				from_vec = to_copy;
+			}
+			memcpy(partial->iov_base +
+			       off_in_tail + to_gap - copied - from_vec,
+			       vec[i].iov_base + off_in_vec,
+			       from_vec);
+
+			gf_log(this->name, GF_LOG_DEBUG,
+			       "uptodate %d bytes at tail. Offset at target(source): %d(%d)",
+			       (int)from_vec,
+			       (int)off_in_tail + to_gap - copied - from_vec,
+			       (int)off_in_vec);
+
+			copied += from_vec;
+			to_copy -= from_vec;
+		}
+		partial->iov_len = off_in_tail + to_gap;
+
+		if (object_alg_should_pad(object)) {
+			int32_t resid = 0;
+			resid = partial->iov_len & (object_alg_blksize(object) - 1);
+			if (resid) {
+				/*
+				 * append a new EOF padding
+				 */
+				local->eof_padding_size =
+					object_alg_blksize(object) - resid;
+
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "set padding size %d",
+				       local->eof_padding_size);
+
+				memset(partial->iov_base + partial->iov_len,
+				       1,
+				       local->eof_padding_size);
+				partial->iov_len += local->eof_padding_size;
+#if DEBUG_CRYPT
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "pad cblock with %d zeros:",
+				       local->eof_padding_size);
+				dump_cblock(this,
+					    (unsigned char *)partial->iov_base +
+					    partial->iov_len - object_alg_blksize(object));
+				check_last_cblock = _gf_true;
+#endif
+			}
+		}
+	}
+	/*
+	 * encrypt the whole block
+	 */
+	encrypt_aligned_iov(object,
+			    partial,
+			    1,
+			    atom->offset_at(frame, object));
+#if DEBUG_CRYPT
+	if (check_last_cblock == _gf_true) {
+		gf_log(this->name, GF_LOG_DEBUG,
+		       "encrypt last cblock with offset %llu",
+		       (unsigned long long)atom->offset_at(frame, object));
+		dump_cblock(this, (unsigned char *)partial->iov_base +
+			    partial->iov_len - object_alg_blksize(object));
+	}
+#endif
+	set_local_io_params_writev(frame, object, atom,
+				   atom->offset_at(frame, object),
+				   iovec_get_size(partial, 1));
+	/*
+	 * write the whole block to disk
+	 */
+	end_writeback_partial_block = dispatch_end_writeback(local->fop);
+	conf->cursor ++;
+	STACK_WIND(frame,
+		   end_writeback_partial_block,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->writev,
+		   local->fd,
+		   partial,
+		   1,
+		   atom->offset_at(frame, object),
+		   local->flags,
+		   local->iobref_data,
+		   local->xdata);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "submit partial block: %d bytes from %d offset",
+	       (int)iovec_get_size(partial, 1),
+	       (int)atom->offset_at(frame, object));
+ exit:
+	return 0;
+}
+
+/*
+ * Perform a (read-)modify-write sequence.
+ * This should be performed only after approval
+ * of upper server-side manager, i.e. the caller
+ * needs to make sure this is his turn to rmw.
+ */
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype)
+{
+	int32_t ret;
+	dict_t *dict;
+	struct rmw_atom *atom;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	atom = atom_by_types(local->active_setup, ltype);
+	/*
+	 * To perform the "read" component of the read-modify-write
+	 * sequence the crypt translator does stack_wind to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 */
+	dict = dict_new();
+	if (!dict) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		return;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		dict_unref(dict);
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   atom->rmw,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   fd,
+		   atom->count_to_uptodate(frame, object), /* count */
+		   atom->offset_at(frame, object), /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	dict_unref(dict);
+}
+
+/*
+ * submit blocks of FULL_ATOM type
+ */
+void submit_full(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	struct rmw_atom *atom = atom_by_types(local->active_setup, FULL_ATOM);
+	uint32_t count; /* total number of full blocks to submit */
+	uint32_t granularity; /* number of blocks to submit in one iteration */
+
+	uint64_t off_in_file; /* start offset in the file, bytes */
+	uint32_t off_in_atom; /* start offset in the atom, blocks */
+	uint32_t blocks_written = 0; /* blocks written for this submit */
+
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_full_block;
+	/*
+	 * Write full blocks by groups of granularity size.
+	 */
+	end_writeback_full_block = dispatch_end_writeback(local->fop);
+
+	if (is_ordered_mode(frame)) {
+		uint32_t skip = has_head_block(conf) ? 1 : 0;
+		count = 1;
+		granularity = 1;
+		/*
+		 * calculate start offset using cursor value;
+		 * here we should take into accout head block,
+		 * which corresponds to cursor value 0.
+		 */
+		off_in_file = atom->offset_at(frame, object) +
+			((conf->cursor - skip) << get_atom_bits(object));
+		off_in_atom = conf->cursor - skip;
+	}
+	else {
+		/*
+		 * in parallel mode
+		 */
+		count = conf->nr_full_blocks;
+		granularity = MAX_IOVEC;
+		off_in_file = atom->offset_at(frame, object);
+		off_in_atom = 0;
+	}
+	while (count) {
+		uint32_t blocks_to_write = count;
+
+		if (blocks_to_write > granularity)
+			blocks_to_write = granularity;
+		if (conf->type == HOLE_ATOM)
+			/*
+			 * reset iovec before encryption
+			 */
+			memset(atom->get_iovec(frame, 0)->iov_base,
+			       0,
+			       get_atom_size(object));
+		/*
+		 * encrypt the group
+		 */
+		encrypt_aligned_iov(object,
+				    atom->get_iovec(frame,
+						    off_in_atom +
+						    blocks_written),
+				    blocks_to_write,
+				    off_in_file + (blocks_written <<
+						   get_atom_bits(object)));
+		
+		set_local_io_params_writev(frame, object, atom,
+		        off_in_file + (blocks_written << get_atom_bits(object)),
+			blocks_to_write <<  get_atom_bits(object));
+
+		conf->cursor += blocks_to_write;
+
+		STACK_WIND(frame,
+			   end_writeback_full_block,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev,
+			   local->fd,
+			   atom->get_iovec(frame, off_in_atom + blocks_written),
+			   blocks_to_write,
+			   off_in_file + (blocks_written << get_atom_bits(object)),
+			   local->flags,
+			   local->iobref_data ? local->iobref_data : local->iobref,
+			   local->xdata);
+
+		gf_log("crypt", GF_LOG_DEBUG, "submit %d full blocks from %d offset",
+		       blocks_to_write,
+		       (int)(off_in_file + (blocks_written << get_atom_bits(object))));
+
+		count -= blocks_to_write;
+		blocks_written += blocks_to_write;
+	}
+	return;
+}
+
+static int32_t rmw_data_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_data_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, TAIL_ATOM));
+}
+
+static int32_t rmw_hole_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_hole_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, TAIL_ATOM));
+}
+
+/*
+ * atom->count_to_uptodate()
+ */
+static uint32_t count_to_uptodate_head(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	if (conf->acount == 1 && conf->off_in_tail)
+		return get_atom_size(object);
+	else
+		/* there is no need to read the whole head block */
+		return conf->off_in_head;
+}
+
+static uint32_t count_to_uptodate_tail(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	/* we need to read the whole tail block */
+	return get_atom_size(object);
+}
+
+static uint32_t count_to_uptodate_data_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_data_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_hole_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_hole_conf(frame), object);
+}
+
+/* atom->get_config() */
+
+static struct avec_config *get_config_data(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->data_conf;
+}
+
+static struct avec_config *get_config_hole(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->hole_conf;
+}
+
+/*
+ * atom->get_iovec()
+ */
+static struct iovec *get_iovec_hole_head(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec;
+}
+
+static struct iovec *get_iovec_hole_full(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0);
+}
+
+static inline struct iovec *get_iovec_hole_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->blocks_in_pool - 1);
+}
+
+static inline struct iovec *get_iovec_data_head(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec;
+}
+
+static inline struct iovec *get_iovec_data_full(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0) + count;
+}
+
+static inline struct iovec *get_iovec_data_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec +
+		(conf->off_in_head ? 1 : 0) +
+		conf->nr_full_blocks;
+}
+
+static struct rmw_atom atoms[LAST_DATA_TYPE][LAST_LOCALITY_TYPE] = {
+	[DATA_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_data_head,
+	  .offset_at = offset_at_data_head,
+	  .offset_in = offset_in_data_head,
+	  .get_iovec = get_iovec_data_head,
+	  .io_size_nopad = io_size_nopad_data_head,
+	  .count_to_uptodate = count_to_uptodate_data_head,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_data_tail,
+	  .offset_at = offset_at_data_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_data_tail,
+	  .io_size_nopad = io_size_nopad_data_tail,
+	  .count_to_uptodate = count_to_uptodate_data_tail,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_data_full,
+	  .offset_in = offset_in_data_full,
+	  .get_iovec = get_iovec_data_full,
+	  .io_size_nopad = io_size_nopad_data_full,
+	  .get_config = get_config_data
+	},
+	[HOLE_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_hole_head,
+	  .offset_at = offset_at_hole_head,
+	  .offset_in = offset_in_hole_head,
+	  .get_iovec = get_iovec_hole_head,
+	  .io_size_nopad = io_size_nopad_hole_head,
+	  .count_to_uptodate = count_to_uptodate_hole_head,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_hole_tail,
+	  .offset_at = offset_at_hole_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_hole_tail,
+	  .io_size_nopad = io_size_nopad_hole_tail,
+	  .count_to_uptodate = count_to_uptodate_hole_tail,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_hole_full,
+	  .offset_in = offset_in_hole_full,
+	  .get_iovec = get_iovec_hole_full,
+	  .io_size_nopad = io_size_nopad_hole_full,
+	  .get_config = get_config_hole
+	}
+};
+
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality)
+{
+	return &atoms[data][locality];
+}
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-common.h b/xlators/encryption/crypt/src/crypt-common.h
new file mode 100644
index 000000000..7c212ad5d
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-common.h
@@ -0,0 +1,141 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_COMMON_H__
+#define __CRYPT_COMMON_H__
+
+#define INVAL_SUBVERSION_NUMBER (0xff)
+#define CRYPT_INVAL_OP   (GF_FOP_NULL)
+
+#define CRYPTO_FORMAT_PREFIX  "trusted.glusterfs.crypt.att.cfmt"
+#define FSIZE_XATTR_PREFIX    "trusted.glusterfs.crypt.att.size"
+#define SUBREQ_PREFIX         "trusted.glusterfs.crypt.msg.sreq"
+#define FSIZE_MSG_PREFIX      "trusted.glusterfs.crypt.msg.size"
+#define DE_MSG_PREFIX         "trusted.glusterfs.crypt.msg.dent"
+#define REQUEST_ID_PREFIX     "trusted.glusterfs.crypt.msg.rqid"
+#define MSGFLAGS_PREFIX       "trusted.glusterfs.crypt.msg.xfgs"
+
+
+/* messages for crypt_open() */
+#define MSGFLAGS_REQUEST_MTD_RLOCK   1 /* take read lock and don't unlock */
+#define MSGFLAGS_REQUEST_MTD_WLOCK   2 /* take write lock and don't unlock */
+
+#define AES_BLOCK_BITS (4) /* AES_BLOCK_SIZE == 1 << AES_BLOCK_BITS */
+
+#define noop   do {; } while (0)
+#define cassert(cond) ({ switch (-1) { case (cond): case 0: break; } })
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+
+/*
+ * Format of file's metadata
+ */
+struct crypt_format {
+	uint8_t loader_id;  /* version of metadata loader */
+	uint8_t versioned[0]; /* file's metadata of specific version */
+} __attribute__((packed));
+
+typedef enum {
+	AES_CIPHER_ALG,
+	LAST_CIPHER_ALG
+} cipher_alg_t;
+
+typedef enum {
+	XTS_CIPHER_MODE,
+	LAST_CIPHER_MODE
+} cipher_mode_t;
+
+typedef enum {
+	MTD_LOADER_V1,
+	LAST_MTD_LOADER
+} mtd_loader_id;
+
+static inline void msgflags_set_mtd_rlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline void msgflags_set_mtd_wlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_rlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_wlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_lock(uint32_t *flags)
+{
+	return  msgflags_check_mtd_rlock(flags) ||
+		msgflags_check_mtd_wlock(flags);
+}
+
+/*
+ * returns number of logical blocks occupied
+ * (maybe partially) by @count bytes
+ * at offset @start.
+ */
+static inline off_t logical_blocks_occupied(uint64_t start, off_t count,
+					    int blkbits)
+{
+	return ((start + count - 1) >> blkbits) - (start >> blkbits) + 1;
+}
+
+/*
+ * are two bytes (represented by offsets @off1
+ * and @off2 respectively) in the same logical
+ * block.
+ */
+static inline int in_same_lblock(uint64_t off1, uint64_t off2,
+				 int blkbits)
+{
+	return off1 >> blkbits == off2 >> blkbits;
+}
+
+static inline void dump_cblock(xlator_t *this, unsigned char *buf)
+{
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "dump cblock: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
+	       (buf)[0],
+	       (buf)[1],
+	       (buf)[2],
+	       (buf)[3],
+	       (buf)[4],
+	       (buf)[5],
+	       (buf)[6],
+	       (buf)[7],
+	       (buf)[8],
+	       (buf)[9],
+	       (buf)[10],
+	       (buf)[11],
+	       (buf)[12],
+	       (buf)[13],
+	       (buf)[14],
+	       (buf)[15]);
+}
+
+#endif /* __CRYPT_COMMON_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-mem-types.h b/xlators/encryption/crypt/src/crypt-mem-types.h
new file mode 100644
index 000000000..799727573
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-mem-types.h
@@ -0,0 +1,43 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __CRYPT_MEM_TYPES_H__
+#define __CRYPT_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_crypt_mem_types_ {
+	gf_crypt_mt_priv = gf_common_mt_end + 1,
+	gf_crypt_mt_inode,
+	gf_crypt_mt_data,
+	gf_crypt_mt_mtd,
+	gf_crypt_mt_loc,
+	gf_crypt_mt_iatt,
+	gf_crypt_mt_key,
+	gf_crypt_mt_iovec,
+	gf_crypt_mt_char,
+};
+
+#endif /* __CRYPT_MEM_TYPES_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
+
+
+
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
new file mode 100644
index 000000000..db2e6d83c
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.c
@@ -0,0 +1,4498 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#include <ctype.h>
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd);
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master);
+static int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				       uint64_t from, off_t size);
+static int32_t load_file_size(call_frame_t *frame, void *cookie,
+			      xlator_t *this, int32_t op_ret, int32_t op_errno,
+			      dict_t *dict, dict_t *xdata);
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype);
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype);
+static void put_one_call_open(call_frame_t *frame);
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this);
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this);
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this);
+static void free_avec(struct iovec *avec, char **pool, int blocks_in_pool);
+static void free_avec_data(crypt_local_t *local);
+static void free_avec_hole(crypt_local_t *local);
+
+static crypt_local_t *crypt_alloc_local(call_frame_t *frame, xlator_t *this,
+					glusterfs_fop_t fop)
+{
+	crypt_local_t *local = NULL;
+
+	local = mem_get0(this->local_pool);
+	if (!local) {
+		gf_log(this->name, GF_LOG_ERROR, "out of memory");
+		return NULL;
+	}
+	local->fop = fop;
+	LOCK_INIT(&local->hole_lock);
+	LOCK_INIT(&local->call_lock);
+	LOCK_INIT(&local->rw_count_lock);
+
+	frame->local = local;
+	return local;
+}
+
+struct crypt_inode_info *get_crypt_inode_info(inode_t *inode, xlator_t *this)
+{
+	int ret;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+
+	ret = inode_ctx_get(inode, this, &value);
+	if (ret == -1) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not get inode info");
+		return NULL;
+	}
+	info = (struct crypt_inode_info *)(long)value;
+	if (info == NULL) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not obtain inode info");
+		return NULL;
+	}
+	return info;
+}
+
+static struct crypt_inode_info *local_get_inode_info(crypt_local_t *local,
+						      xlator_t *this)
+{
+	if (local->info)
+		return local->info;
+	local->info = get_crypt_inode_info(local->fd->inode, this);
+	return local->info;
+}
+
+static struct crypt_inode_info *alloc_inode_info(crypt_local_t *local,
+						 loc_t *loc)
+{
+	struct crypt_inode_info *info;
+
+	info = GF_CALLOC(1, sizeof(*info), gf_crypt_mt_inode);
+	if (!info) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		gf_log ("crypt", GF_LOG_WARNING,
+			"Can not allocate inode info");
+		return NULL;
+	}
+	memset(info, 0, sizeof(*info));
+#if DEBUG_CRYPT
+	info->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!info->loc) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not allocate loc");
+		GF_FREE(info);
+		return NULL;
+	}
+	if (loc_copy(info->loc, loc)){
+		GF_FREE(info->loc);
+		GF_FREE(info);
+		return NULL;
+	}
+#endif /* DEBUG_CRYPT */
+
+	local->info = info;
+	return info;
+}
+
+static void free_inode_info(struct crypt_inode_info *info)
+{
+#if DEBUG_CRYPT
+	loc_wipe(info->loc);
+	GF_FREE(info->loc);
+#endif
+	memset(info, 0, sizeof(*info));
+	GF_FREE(info);
+}
+
+int crypt_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        if (!inode_ctx_del (inode, this, &ctx_addr))
+                free_inode_info((struct crypt_inode_info *)(long)ctx_addr);
+        return 0;
+}
+
+#if DEBUG_CRYPT
+static void check_read(call_frame_t *frame, xlator_t *this, int32_t read,
+		       struct iovec *vec, int32_t count, struct iatt *stbuf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = get_object_cinfo(local->info);
+	struct avec_config *conf = &local->data_conf;
+	uint32_t resid = stbuf->ia_size & (object_alg_blksize(object) - 1);
+
+	if (read <= 0)
+		return;
+	if (read != iovec_get_size(vec, count))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"op_ret differs from amount of read bytes");
+
+	if (object_alg_should_pad(object) && (read & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (!= 0 mod(cblock size))");
+
+	if (conf->aligned_offset + read >
+	    stbuf->ia_size + (resid ? object_alg_blksize(object) - resid : 0))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (too large))");
+
+}
+
+#define PT_BYTES_TO_DUMP (32)
+static void dump_plain_text(crypt_local_t *local, struct iovec *avec)
+{
+	int32_t to_dump;
+	char str[PT_BYTES_TO_DUMP + 1];
+
+	if (!avec)
+		return;
+	to_dump = avec->iov_len;
+	if (to_dump > PT_BYTES_TO_DUMP)
+		to_dump = PT_BYTES_TO_DUMP;
+	memcpy(str, avec->iov_base, to_dump);
+	memset(str + to_dump, '0', 1);
+	gf_log("crypt", GF_LOG_DEBUG, "Read file: %s", str);
+}
+
+static int32_t data_conf_invariant(struct avec_config *conf)
+{
+	return conf->acount ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		conf->nr_full_blocks;
+}
+
+static int32_t hole_conf_invariant(struct avec_config *conf)
+{
+	return conf->blocks_in_pool ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		!!has_full_blocks(conf);
+}
+
+static void crypt_check_conf(struct avec_config *conf)
+{
+	int32_t ret = 0;
+	const char *msg;
+
+	switch (conf->type) {
+	case DATA_ATOM:
+		msg = "data";
+		ret = data_conf_invariant(conf);
+		break;
+	case HOLE_ATOM:
+		msg = "hole";
+		ret = hole_conf_invariant(conf);
+		break;
+	default:
+		msg = "unknown";
+	}
+	if (!ret)
+		gf_log("crypt", GF_LOG_DEBUG, "bad %s conf", msg);
+}
+
+static void check_buf(call_frame_t *frame, xlator_t *this, struct iatt *buf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	uint64_t local_file_size;
+
+	switch(local->fop) {
+	case GF_FOP_FTRUNCATE:
+		return;
+	case GF_FOP_WRITE:
+		local_file_size = local->new_file_size;
+		break;
+	case GF_FOP_READ:
+		if (parent_is_crypt_xlator(frame, this))
+			return;
+		local_file_size = local->cur_file_size;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad file operation");
+		return;
+	}
+	if (buf->ia_size != round_up(local_file_size,
+					 object_alg_blksize(object)))
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "bad ia_size in buf (%llu), should be %llu",
+		       (unsigned long long)buf->ia_size,
+		       (unsigned long long)round_up(local_file_size,
+						    object_alg_blksize(object)));
+}
+
+#else
+#define check_read(frame, this, op_ret, vec, count, stbuf) noop
+#define dump_plain_text(local, avec) noop
+#define crypt_check_conf(conf) noop
+#define check_buf(frame, this, buf) noop
+#endif /* DEBUG_CRYPT */
+
+/*
+ * Pre-conditions:
+ * @vec represents a ciphertext of expanded size and
+ * aligned offset.
+ *
+ * Compound a temporal vector @avec with block-aligned
+ * components, decrypt and fix it up to represent a chunk
+ * of data corresponding to the original size and offset.
+ * Pass the result to the next translator.
+ */
+int32_t crypt_readv_cbk(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			struct iovec *vec,
+			int32_t count,
+			struct iatt *stbuf,
+			struct iobref *iobref,
+			dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *avec;
+	uint32_t i;
+	uint32_t to_vec;
+	uint32_t to_user;
+
+	check_buf(frame, this, stbuf);
+	check_read(frame, this, op_ret, vec, count, stbuf);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->iobref = iobref_ref(iobref);
+
+	local->buf = *stbuf;
+	local->buf.ia_size = local->cur_file_size;
+
+	if (op_ret <= 0 || count == 0 || vec[0].iov_len == 0)
+		goto put_one_call;
+
+	if (conf->orig_offset >= local->cur_file_size) {
+		local->op_ret = 0;
+		goto put_one_call;
+	}
+	/* 
+	 * correct config params with real file size
+	 * and actual amount of bytes read
+	 */
+	set_config_offsets(frame, this,
+			   conf->orig_offset, op_ret, DATA_ATOM, 0);
+
+	if (conf->orig_offset + conf->orig_size > local->cur_file_size)
+		conf->orig_size = local->cur_file_size - conf->orig_offset;
+	/*
+	 * calculate amount of data to be returned
+	 * to user.
+	 */
+	to_user = op_ret;
+	if (conf->aligned_offset + to_user <= conf->orig_offset) {
+		gf_log(this->name, GF_LOG_WARNING, "Incomplete read");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	to_user -= (conf->aligned_offset - conf->orig_offset);
+
+	if (to_user > conf->orig_size)
+		to_user = conf->orig_size;
+	local->rw_count = to_user;
+
+	op_errno = set_config_avec_data(this, local,
+					conf, object, vec, count);
+	if (op_errno) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
+		goto put_one_call;
+	}
+	avec = conf->avec;
+#if DEBUG_CRYPT
+	if (conf->off_in_tail != 0 &&
+	    conf->off_in_tail < object_alg_blksize(object) &&
+	    object_alg_should_pad(object))
+		gf_log(this->name, GF_LOG_DEBUG, "Bad offset in tail %d",
+		       conf->off_in_tail);
+	if (iovec_get_size(vec, count) != 0 &&
+	    in_same_lblock(conf->orig_offset + iovec_get_size(vec, count) - 1,
+			   local->cur_file_size - 1,
+			   object_alg_blkbits(object))) {
+		gf_log(this->name, GF_LOG_DEBUG, "Compound last cblock");
+		dump_cblock(this,
+			    (unsigned char *)(avec[conf->acount - 1].iov_base) +
+			    avec[conf->acount - 1].iov_len - object_alg_blksize(object));
+		dump_cblock(this,
+			    (unsigned char *)(vec[count - 1].iov_base) +
+			    vec[count - 1].iov_len - object_alg_blksize(object));
+	}
+#endif
+	decrypt_aligned_iov(object, avec,
+			    conf->acount, conf->aligned_offset);
+	/*
+	 * pass proper plain data to user
+	 */
+	avec[0].iov_base += (conf->aligned_offset - conf->orig_offset);
+	avec[0].iov_len  -= (conf->aligned_offset - conf->orig_offset);
+
+	to_vec = to_user;
+	for (i = 0; i < conf->acount; i++) {
+		if (avec[i].iov_len > to_vec)
+			avec[i].iov_len = to_vec;
+		to_vec -= avec[i].iov_len;
+	}
+ put_one_call:
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t do_readv(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			dict_t *dict,
+			dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->cur_file_size = data_to_uint64(data);
+	
+	get_one_call(frame);
+	STACK_WIND(frame,
+		   crypt_readv_cbk,
+		   FIRST_CHILD (this),
+		   FIRST_CHILD (this)->fops->readv,
+		   local->fd,
+		   /*
+		    * FIXME: read amount can be reduced
+		    */
+		   local->data_conf.expanded_size,
+		   local->data_conf.aligned_offset,
+		   local->flags,
+		   local->xdata);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call(frame);
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t crypt_readv_finodelk_cbk(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size
+	 */
+	STACK_WIND(frame,
+		   do_readv,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return  0;
+ error:
+	fd_unref(local->fd);
+	if (local->xdata)
+		dict_unref(local->xdata);
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    op_errno,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t readv_trivial_completion(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					struct iatt *buf,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "stat failed (%d)", op_errno);
+		goto error;
+	}
+	local->buf = *buf;
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;	
+ error:
+	STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno,
+			    NULL, 0, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t crypt_readv(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    size_t size,
+		    off_t offset,
+		    uint32_t flags, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "reading %d bytes from offset %llu",
+	       (int)size, (long long)offset);
+	if (parent_is_crypt_xlator(frame, this))
+		gf_log("crypt", GF_LOG_DEBUG, "parent is crypt");
+#endif	
+	local = crypt_alloc_local(frame, this, GF_FOP_READ);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	if (size == 0)
+		goto trivial;
+
+	local->fd = fd_ref(fd);
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	set_config_offsets(frame, this, offset, size,
+			   DATA_ATOM, 0);
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * We are called by crypt_writev (or cypt_ftruncate)
+		 * to perform the "read" component of the read-modify-write
+		 * (or read-prune-write) sequence for some atom;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 *
+		 * Retrieve current file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		get_one_call(frame);
+		STACK_WIND(frame,
+			   crypt_readv_cbk,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->readv,
+			   local->fd,
+			   /*
+			    * FIXME: read amount can be reduced
+			    */
+			   local->data_conf.expanded_size,
+			   local->data_conf.aligned_offset,
+			   flags,
+			   NULL);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_readv_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   readv_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom,
+				off_t io_offset,
+				uint32_t io_size)
+{
+	crypt_local_t *local = frame->local;
+
+	local->io_offset = io_offset;
+	local->io_size = io_size;
+
+	local->io_offset_nopad =
+		atom->offset_at(frame, object) + atom->offset_in(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad offset to %llu",
+	       (unsigned long long)local->io_offset_nopad);
+
+	local->io_size_nopad = atom->io_size_nopad(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad size to %llu",
+	       (unsigned long long)local->io_size_nopad);
+
+	local->update_disk_file_size = 0;
+	/*
+	 * NOTE: eof_padding_size is 0 for all full atoms;
+	 * For head and tail atoms it will be set up at rmw_partial block()
+	 */
+	local->new_file_size = local->cur_file_size;
+
+	if (local->io_offset_nopad + local->io_size_nopad > local->cur_file_size) {
+
+		local->new_file_size = local->io_offset_nopad + local->io_size_nopad;
+
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "set new file size to %llu",
+		       (unsigned long long)local->new_file_size);
+
+		local->update_disk_file_size = 1;
+	}
+}
+
+void set_local_io_params_ftruncate(call_frame_t *frame,
+				   struct object_cipher_info *object)
+{
+	uint32_t resid;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	resid = conf->orig_offset & (object_alg_blksize(object) - 1);
+	if (resid) {
+		local->eof_padding_size =
+			object_alg_blksize(object) - resid;
+		local->new_file_size = conf->aligned_offset;
+		local->update_disk_file_size = 0;
+		/*
+		 * file size will be updated
+		 * in the ->writev() stack,
+		 * when submitting file tail
+		 */
+	}
+	else {	
+		local->eof_padding_size = 0;
+		local->new_file_size = conf->orig_offset;
+		local->update_disk_file_size = 1;
+		/*
+		 * file size will be updated
+		 * in this ->ftruncate stack
+		 */
+	}
+}
+
+static inline void submit_head(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, HEAD_ATOM);
+}
+
+static inline void submit_tail(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, TAIL_ATOM);
+}
+
+static void submit_hole(call_frame_t *frame, xlator_t *this)
+{
+	/*
+	 * hole conversion always means
+	 * appended write and goes in ordered fashion
+	 */
+	do_ordered_submit(frame, this, HOLE_ATOM);
+}
+
+static void submit_data(call_frame_t *frame, xlator_t *this)
+{
+	if (is_ordered_mode(frame)) {
+		do_ordered_submit(frame, this, DATA_ATOM);
+		return;
+	}
+	gf_log("crypt", GF_LOG_WARNING, "Bad submit mode");
+	get_nr_calls(frame, nr_calls_data(frame));
+	do_parallel_submit(frame, this, DATA_ATOM);
+	return;
+}
+
+/*
+ * heplers called by writev_cbk, fruncate_cbk in ordered mode
+ */
+
+static inline int32_t should_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	return conf->avec != NULL;
+}
+
+static inline int32_t should_resume_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	if (local->fop == GF_FOP_WRITE && has_tail_block(conf))
+		/*
+		 * Don't submit a part of hole, which
+		 * fits into a data block:
+ 		 * this part of hole will be converted
+		 * as a gap filled by zeros in data head
+		 * block.
+		 */
+		return conf->cursor < conf->acount - 1;
+	else
+		return conf->cursor < conf->acount;
+}
+
+static inline int32_t should_resume_submit_data(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	if (is_ordered_mode(frame))
+		return conf->cursor < conf->acount;
+	/*
+	 * parallel writes
+	 */
+	return 0;
+}
+
+static inline int32_t should_submit_data_after_hole(crypt_local_t *local)
+{
+	return local->data_conf.avec != NULL;
+}
+
+static void update_local_file_params(call_frame_t *frame,
+				     xlator_t *this,
+				     struct iatt *prebuf,
+				     struct iatt *postbuf)
+{
+	crypt_local_t *local = frame->local;
+
+	check_buf(frame, this, postbuf);
+
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	local->prebuf.ia_size  = local->cur_file_size;
+	local->postbuf.ia_size = local->new_file_size;
+
+	local->cur_file_size = local->new_file_size;	
+}
+
+static int32_t end_writeback_writev(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    struct iatt *prebuf,
+				    struct iatt *postbuf,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret <= 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "writev iteration failed");
+		goto put_one_call;
+	}
+	/*
+	 * op_ret includes paddings (atom's head, atom's tail and EOF)
+	 */
+	if (op_ret < local->io_size) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Incomplete writev iteration");
+		goto put_one_call;
+	}
+	op_ret -= local->eof_padding_size;
+	local->op_ret = op_ret;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local)) {
+
+		LOCK(&local->rw_count_lock);
+		local->rw_count += op_ret;
+		UNLOCK(&local->rw_count_lock);
+
+		if (should_resume_submit_data(frame))
+			submit_data(frame, this);
+	}
+	else {
+		/*
+		 * hole conversion is going on;
+		 * don't take into account written zeros
+		 */
+		if (should_resume_submit_hole(local))
+			submit_hole(frame, this);
+
+		else if (should_submit_data_after_hole(local))
+			submit_data(frame, this);
+	}
+ put_one_call:
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+#define crypt_writev_cbk end_writeback_writev
+
+#define HOLE_WRITE_CHUNK_BITS 12
+#define HOLE_WRITE_CHUNK_SIZE (1 << HOLE_WRITE_CHUNK_BITS)
+
+/*
+ * Convert hole of size @size at offset @off to
+ * zeros and prepare respective iovecs for submit.
+ * The hole lock should be held.
+ *
+ * Pre-conditions:
+ * @local->file_size is set and valid.
+ */
+int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				uint64_t off, off_t size)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, off, size, HOLE_ATOM, 1);
+
+	ret = set_config_avec_hole(this, local,
+				   &local->hole_conf, object, local->fop);
+	crypt_check_conf(&local->hole_conf);
+
+	return ret;
+}
+
+/*
+ * prepare for submit @count bytes at offset @from
+ */
+int32_t prepare_for_submit_data(call_frame_t *frame, xlator_t *this,
+				off_t from, int32_t size, struct iovec *vec,
+				int32_t vec_count, int32_t setup_gap)
+{
+	uint32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, from, size,
+			   DATA_ATOM, setup_gap);
+
+	ret = set_config_avec_data(this, local,
+				   &local->data_conf, object, vec, vec_count);
+	crypt_check_conf(&local->data_conf);
+
+	return ret;
+}
+
+static void free_avec(struct iovec *avec,
+		      char **pool, int blocks_in_pool)
+{
+	if (!avec)
+		return;
+	GF_FREE(pool);
+	GF_FREE(avec);
+}
+
+static void free_avec_data(crypt_local_t *local)
+{
+	return free_avec(local->data_conf.avec,
+			 local->data_conf.pool,
+			 local->data_conf.blocks_in_pool);
+}
+
+static void free_avec_hole(crypt_local_t *local)
+{
+	return free_avec(local->hole_conf.avec,
+			 local->hole_conf.pool,
+			 local->hole_conf.blocks_in_pool);
+}
+
+
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (has_head_block(conf))
+		submit_head(frame, this);
+
+	if (has_full_blocks(conf))
+		submit_full(frame, this);
+
+	if (has_tail_block(conf))
+		submit_tail(frame, this);
+	return;
+}
+
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (should_submit_head_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_head(frame, this);
+	}
+	else if (should_submit_full_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_full(frame, this);
+	}
+	else if (should_submit_tail_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_tail(frame, this);
+	}
+	else
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "nothing has been submitted in ordered mode");
+	return;
+}
+
+static int32_t do_writev(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	set_gap_at_end(frame, object, &local->data_conf, DATA_ATOM);
+
+	if (local->cur_file_size < local->data_conf.orig_offset) {
+		/*
+		 * Set up hole config
+		 */
+		op_errno = prepare_for_submit_hole(frame,
+			       this,
+			       local->cur_file_size,
+			       local->data_conf.orig_offset - local->cur_file_size);
+		if (op_errno) {
+			local->op_ret = -1;
+			local->op_errno = op_errno;
+			goto error;
+		}
+	}
+	if (should_submit_hole(local))
+		submit_hole(frame, this);
+	else 
+		submit_data(frame, this);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+static int32_t crypt_writev_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_writev,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;	
+}
+
+static int32_t writev_trivial_completion(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 struct iatt *buf,
+					 dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+	
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+int crypt_writev(call_frame_t *frame,
+		 xlator_t *this,
+		 fd_t *fd,
+		 struct iovec *vec,
+		 int32_t count,
+		 off_t offset,
+		 uint32_t flags,
+		 struct iobref *iobref,
+		 dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+#if DEBUG_CRYPT
+	gf_log ("crypt", GF_LOG_DEBUG, "writing %d bytes from offset %llu",
+		(int)iovec_get_size(vec, count), (long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_WRITE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	if (iobref)
+		local->iobref = iobref_ref(iobref);
+	/*
+	 * to update real file size on the server
+	 */
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (iovec_get_size(vec, count) == 0)
+		goto trivial;
+
+	ret = prepare_for_submit_data(frame, this, offset,
+				      iovec_get_size(vec, count),
+				      vec, count, 0 /* don't setup gup
+						       in tail: we don't
+						       know file size yet */);
+	if (ret)
+		goto error;
+
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * we are called by shinking crypt_ftruncate(),
+		 * which doesn't perform hole conversion;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 */
+
+		/*
+		 * extract file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		submit_data(frame, this);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	/*
+	 * lock the file and retrieve its size
+	 */
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_writev_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   writev_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->iobref)
+		iobref_unref(iobref);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(writev, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t prepare_for_prune(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	set_config_offsets(frame, this,
+			   offset,
+			   0, /* count */
+			   DATA_ATOM,
+			   0 /* since we prune, there is no
+				gap in tail to uptodate */);
+	return 0;
+}
+
+/*
+ * Finish the read-prune-modify sequence
+ *
+ * Can be invoked as
+ * 1) ->ftruncate_cbk() for cblock-aligned, or trivial prune
+ * 2) ->writev_cbk() for non-cblock-aligned prune
+ */
+
+static int32_t prune_complete(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      struct iatt *prebuf,
+			      struct iatt *postbuf,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as ->ftruncate_cbk()
+ *
+ * Perform the "write" component of the
+ * read-prune-write sequence.
+ *
+ * submuit the rest of the file
+ */
+static int32_t prune_submit_file_tail(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno,
+				      struct iatt *prebuf,
+				      struct iatt *postbuf,
+				      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	dict_t *dict;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	dict = dict_new();
+	if (!dict) {
+		op_errno = ENOMEM;
+		goto error;
+	}
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+	local->new_file_size = conf->orig_offset;
+
+	/*
+	 * The rest of the file is a partial block and, hence,
+	 * should be written via RMW sequence, so the crypt xlator
+	 * does STACK_WIND to itself.
+	 *
+	 * Pass current file size to crypt_writev()
+	 */
+	op_errno = dict_set(dict,
+			    FSIZE_XATTR_PREFIX,
+			    data_from_uint64(local->cur_file_size));
+	if (op_errno) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "can not set key to update file size");
+		dict_unref(dict);
+		goto error;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "passing current file size (%llu) to crypt_writev",
+	       (unsigned long long)local->cur_file_size);
+	/*
+	 * Padding will be filled with
+	 * zeros by rmw_partial_block()
+	 */
+	STACK_WIND(frame,
+		   prune_complete,
+		   this,
+		   this->fops->writev, /* crypt_writev */
+		   local->fd,
+		   &local->vec,
+		   1,
+		   conf->aligned_offset, /* offset to write from */
+		   0,
+		   local->iobref,
+		   dict);
+
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as a callback of ->writev() invoked in behalf
+ * of ftruncate(): it can be
+ * 1) ordered writes issued by hole conversion in the case of
+ *    expanded truncate, or
+ * 2) an rmw partial data block issued by non-cblock-aligned
+ * prune.
+ */
+int32_t end_writeback_ftruncate(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				struct iatt *prebuf,
+				struct iatt *postbuf,
+				dict_t *xdata)
+{
+	crypt_local_t  *local = frame->local;
+	/*
+	 * if nothing has been written,
+	 * then it must be an error
+	 */
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local))
+		/* case (2) */
+		goto put_one_call;
+	/* case (1) */
+	if (should_resume_submit_hole(local))
+		submit_hole(frame, this);
+	/*
+	 * case of hole, when we should't resume
+	 */
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform prune and write components of the
+ * read-prune-write sequence.
+ *
+ * Called as ->readv_cbk()
+ *
+ * Pre-conditions:
+ * @vec contains the latest atom of the file
+ * (plain text)
+ */
+static int32_t prune_write(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno,
+			   struct iovec *vec,
+			   int32_t count,
+			   struct iatt *stbuf,
+			   struct iobref *iobref,
+			   dict_t *xdata)
+{
+	int32_t i;
+	size_t to_copy;
+	size_t copied = 0;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret == -1)
+		goto put_one_call;
+
+	/*
+	 * At first, uptodate head block
+	 */
+	if (iovec_get_size(vec, count) < conf->off_in_head) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Failed to uptodate head block for prune");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	local->vec.iov_len = conf->off_in_head;
+	local->vec.iov_base = GF_CALLOC(1, local->vec.iov_len,
+					gf_crypt_mt_data);
+
+	if (local->vec.iov_base == NULL) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+	}
+	for (i = 0; i < count; i++) {
+		to_copy = vec[i].iov_len;
+		if (to_copy > local->vec.iov_len - copied)
+			to_copy = local->vec.iov_len - copied;
+
+		memcpy((char *)local->vec.iov_base + copied,
+		       vec[i].iov_base,
+		       to_copy);
+		copied += to_copy;
+		if (copied == local->vec.iov_len)
+			break;
+	}
+	/*
+	 * perform prune with aligned offset
+	 * (i.e. at this step we prune a bit
+	 * more then it is needed
+	 */
+	STACK_WIND(frame,
+		   prune_submit_file_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->ftruncate,
+		   local->fd,
+		   conf->aligned_offset,
+		   local->xdata);
+	return 0;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform a read-prune-write sequence
+ */
+int32_t read_prune_write(call_frame_t *frame, xlator_t *this)
+{
+	int32_t ret = 0;
+	dict_t *dict = NULL;
+	crypt_local_t  *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_local_io_params_ftruncate(frame, object);
+	get_one_call_nolock(frame);
+
+	if ((conf->orig_offset & (object_alg_blksize(object) - 1)) == 0) {
+		/*
+		 * cblock-aligned prune:
+		 * we don't need read and write components,
+		 * just cut file body
+		 */
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "prune without RMW (at offset %llu",
+		       (unsigned long long)conf->orig_offset);
+
+		STACK_WIND(frame,
+			   prune_complete,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->ftruncate,
+			   local->fd,
+			   conf->orig_offset,
+			   local->xdata);
+		return 0;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "prune with RMW (at offset %llu",
+	       (unsigned long long)conf->orig_offset);	
+	/*
+	 * We are about to perform the "read" component of the
+	 * read-prune-write sequence. It means that we need to
+	 * read encrypted data from disk and decrypt it.
+	 * So, the crypt translator does STACK_WIND to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 
+	 */
+	dict = dict_new();
+	if (!dict) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		ret = ENOMEM;
+		goto exit;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   prune_write,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   local->fd,
+		   get_atom_size(object), /* bytes to read */
+		   conf->aligned_offset, /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	if (dict)
+		dict_unref(dict);
+	return ret;
+}
+
+/*
+ * File prune is more complicated than expand.
+ * First we need to read the latest atom to not lose info
+ * needed for proper update. Also we need to make sure that
+ * every component of read-prune-write sequence leaves data
+ * consistent
+ *
+ * Non-cblock aligned prune is performed as read-prune-write
+ * sequence:
+ *
+ * 1) read the latest atom;
+ * 2) perform cblock-aligned prune
+ * 3) issue a write request for the end-of-file
+ */
+int32_t prune_file(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	int32_t ret;
+
+	ret = prepare_for_prune(frame, this, offset);
+	if (ret)
+		return ret;
+	return read_prune_write(frame, this);
+}
+
+int32_t expand_file(call_frame_t *frame, xlator_t *this,
+		    uint64_t offset)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+
+	ret = prepare_for_submit_hole(frame, this,
+				      local->old_file_size,
+				      offset - local->old_file_size);
+	if (ret)
+		return ret;
+	submit_hole(frame, this);
+	return 0;
+}
+
+static int32_t ftruncate_trivial_completion(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    struct iatt *buf,
+					    dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t do_ftruncate(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    dict_t *dict,
+			    dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	if (local->data_conf.orig_offset == local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "trivial ftruncate (current file size %llu)",
+		       (unsigned long long)local->cur_file_size);
+#endif
+		goto trivial;
+	}
+	else if (local->data_conf.orig_offset < local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "prune from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = prune_file(frame,
+				      this,
+				      local->data_conf.orig_offset);
+	}
+	else {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "expand from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = expand_file(frame,
+				       this,
+				       local->data_conf.orig_offset);
+	}
+	if (op_errno)
+		goto error;
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   ftruncate_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	/*
+	 * finish with ftruncate
+	 */
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_finodelk_cbk(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_ftruncate,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * ftruncate is performed in 2 steps:
+ * . recieve file size;
+ * . expand or prune file.
+ */
+static int32_t crypt_ftruncate(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       off_t offset,
+			       dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FTRUNCATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	local->data_conf.orig_offset = offset;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_ftruncate_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(ftruncate, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+/* ->flush_cbk() */
+int32_t truncate_end(call_frame_t *frame,
+		     void *cookie,
+		     xlator_t *this,
+		     int32_t op_ret,
+		     int32_t op_errno,
+		     dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_UNWIND_STRICT(truncate,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local->xdata);
+	return 0;
+}
+
+/* ftruncate_cbk() */
+int32_t truncate_flush(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iatt *prebuf,
+		       struct iatt *postbuf,
+		       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *fd = local->fd;
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	STACK_WIND(frame,
+		   truncate_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   fd,
+		   NULL);
+	fd_unref(fd);
+	return 0;
+}
+
+/*
+ * is called as ->open_cbk()
+ */
+static int32_t truncate_begin(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0) {
+		fd_unref(fd);
+		STACK_UNWIND_STRICT(truncate,
+				    frame,
+				    op_ret,
+				    op_errno, NULL, NULL, NULL);
+		return 0;
+	}
+	/*
+	 * crypt_truncate() is implemented via crypt_ftruncate(),
+	 * so the crypt xlator does STACK_WIND to itself here
+	 */
+	STACK_WIND(frame,
+		   truncate_flush,
+		   this,
+		   this->fops->ftruncate, /* crypt_ftruncate */
+		   fd,
+		   local->offset,
+		   NULL);
+	return 0;
+}
+
+/*
+ * crypt_truncate() is implemented via crypt_ftruncate() as a
+ * sequence crypt_open() - crypt_ftruncate() - truncate_flush()
+ */
+int32_t crypt_truncate(call_frame_t *frame,
+		       xlator_t *this,
+		       loc_t *loc,
+		       off_t offset,
+		       dict_t *xdata)
+{
+	fd_t *fd;
+	crypt_local_t *local;
+
+#if DEBUG_CRYPT
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "truncate file %s at offset %llu",
+	       loc->path, (unsigned long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_TRUNCATE);
+	if (!local)
+		goto error;
+
+	fd = fd_create(loc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->offset = offset;
+	local->xdata = xdata;
+	STACK_WIND(frame,
+		   truncate_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   loc,
+		   O_RDWR,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(truncate, frame, -1, EINVAL, NULL, NULL, NULL);
+	return 0;
+}
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_WRITE:
+		return end_writeback_writev;
+	case GF_FOP_FTRUNCATE:
+		return end_writeback_ftruncate;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad wb operation %d", fop);
+		return NULL;
+	}
+}
+
+/*
+ * true, if the caller needs metadata string
+ */
+static int32_t is_custom_mtd(dict_t *xdata)
+{
+	data_t *data;
+	uint32_t flags;
+
+	if (!xdata)
+		return 0;
+
+	data = dict_get(xdata, MSGFLAGS_PREFIX);
+	if (!data)
+		return 0;
+	if (data->len != sizeof(uint32_t)) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "Bad msgflags size (%d)", data->len);
+		return -1;
+	}
+	flags = *((uint32_t *)data->data);
+	return msgflags_check_mtd_lock(&flags);
+}
+
+static int32_t crypt_open_done(call_frame_t *frame,
+			       void *cookie,
+			       xlator_t *this,
+			       int32_t op_ret,
+			       int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret < 0)
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+	put_one_call_open(frame);
+	return 0;
+}
+
+static void crypt_open_tail(call_frame_t *frame, xlator_t *this)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_open_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+}
+
+/*
+ * load private inode info at open time
+ * called as ->fgetxattr_cbk()
+ */
+static int load_mtd_open(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	int32_t ret;
+	gf_boolean_t upload_info;
+	data_t *mtd;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	/*
+	 * first, check for cached info
+	 */
+	ret = inode_ctx_get(local->fd->inode, this, &value);
+	if (ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Inode info expected, but not found");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * info has been found in the cache
+		 */
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info hasn't been found in the cache.
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			goto exit;
+		}
+		init_inode_info_head(info, local->fd);
+		upload_info = _gf_true;
+	}
+	/*
+	 * extract metadata
+	 */
+	mtd = dict_get(dict, CRYPTO_FORMAT_PREFIX);
+	if (!mtd) {
+		local->op_ret = -1;
+		local->op_errno = ENOENT;
+		gf_log (this->name, GF_LOG_WARNING,
+			"Format string wasn't found");
+		goto exit;
+	}
+	/*
+	 * authenticate metadata against the path
+	 */	
+	ret = open_format((unsigned char *)mtd->data,
+			  mtd->len,
+			  local->loc,
+			  info,
+			  get_master_cinfo(priv),
+			  local,
+			  upload_info);
+	if (ret) {
+		local->op_ret = -1;
+		local->op_errno = ret;
+		goto exit;
+	}
+	if (upload_info) {
+		ret = init_inode_info_tail(info, get_master_cinfo(priv));
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;	
+		}
+		ret = inode_ctx_put(local->fd->inode,
+				    this, (uint64_t)(long)info);
+		if (ret == -1) {
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+	}
+	if (local->custom_mtd) {
+		/*
+		 * pass the metadata string to the customer
+		 */
+		ret = dict_set_static_bin(local->xdata,
+					  CRYPTO_FORMAT_PREFIX,
+					  mtd->data,
+					  mtd->len);
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;
+		}
+	}
+ exit:
+	if (!local->custom_mtd)
+		crypt_open_tail(frame, this);
+	else
+		put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open_finodelk_cbk(call_frame_t *frame,
+				       void *cookie,
+				       xlator_t *this,
+				       int32_t op_ret,
+				       int32_t op_errno,
+				       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING, "finodelk (LOCK) failed");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   load_mtd_open,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   CRYPTO_FORMAT_PREFIX,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+/*
+ * verify metadata against the specified pathname
+ */
+static int32_t crypt_open_cbk(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	else if (local->custom_mtd){
+		local->xdata = dict_new();
+		if (!local->xdata) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			gf_log ("crypt", GF_LOG_ERROR,
+				"Can not get new dict for mtd string");
+			goto exit;
+		}
+	}
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = local->custom_mtd ? F_WRLCK : F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_open_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc,
+			  int32_t flags,
+			  fd_t *fd,
+			  dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_OPEN);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc) {
+		ret = ENOMEM;
+		goto error;
+	}
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	ret = is_custom_mtd(xdata);
+	if (ret < 0) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+		ret = EINVAL;
+		goto error;
+	}
+	local->custom_mtd = ret;
+
+	if ((flags & O_ACCMODE) == O_WRONLY)
+		/*
+		 * we can't open O_WRONLY, because
+		 * we need to do read-modify-write
+		 */
+		flags = (flags & ~O_ACCMODE) | O_RDWR;
+	/*
+	 * Make sure that out translated offsets
+	 * and counts won't be ignored
+	 */
+	flags &= ~O_APPEND;
+	get_one_call_nolock(frame);
+	STACK_WIND(frame,
+		   crypt_open_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->open,
+		   loc,
+		   flags,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(open,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct object_cipher_info *object = &info->cinfo;
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "Init inode info for object %s",
+	       uuid_utoa(info->oid));
+#endif
+	ret = data_cipher_algs[object->o_alg][object->o_mode].set_private(info,
+									master);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set private info failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * Init inode info at ->create() time
+ */
+static void init_inode_info_create(struct crypt_inode_info *info,
+				   struct master_cipher_info *master,
+				   data_t *data)
+{
+	struct object_cipher_info *object;
+
+	info->nr_minor = CRYPT_XLATOR_ID;
+	memcpy(info->oid, data->data, data->len);
+
+	object = &info->cinfo;
+
+	object->o_alg        = master->m_alg;
+	object->o_mode       = master->m_mode;
+	object->o_block_bits = master->m_block_bits;
+	object->o_dkey_size  = master->m_dkey_size;
+}
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd)
+{
+	memcpy(info->oid, fd->inode->gfid, sizeof(uuid_t));
+}
+
+static int32_t crypt_create_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno, dict_t *xdata)
+{
+	crypt_private_t *priv = this->private;
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0) {
+		free_inode_info(info);
+		goto unwind;
+	}
+	op_errno = init_inode_info_tail(info, get_master_cinfo(priv));
+	if (op_errno) {
+		op_ret = -1;
+		free_inode_info(info);
+		goto unwind;
+	}
+	/*
+	 * FIXME: drop major subversion number
+	 */
+	op_ret = inode_ctx_put(local->fd->inode, this, (uint64_t)(long)info);
+	if (op_ret == -1) {
+		op_errno = EIO;
+		free_inode_info(info);
+		goto unwind;
+	}
+ unwind:
+	free_format(local);
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int crypt_create_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	dict_unref(local->xattr);
+
+	if (op_ret < 0)
+		goto error;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_create_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	free_inode_info(local->info);
+	free_format(local);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_create_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+
+	STACK_WIND(frame,
+		   crypt_create_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   local->fd,
+		   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+		   0,
+		   NULL);
+	return 0;
+ error: 
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+	if (local->xdata)
+		dict_unref(local->xdata);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * Create and store crypt-specific format on disk;
+ * Populate cache with private inode info
+ */
+static int32_t crypt_create_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				fd_t *fd,
+				inode_t *inode,
+				struct iatt *buf,
+				struct iatt *preparent,
+				struct iatt *postparent,
+				dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);	
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND(frame,
+		   crypt_create_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);	
+	return 0;
+ error:
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_create(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t flags,
+			    mode_t mode,
+			    mode_t umask,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	int ret;
+	data_t *data;
+	crypt_local_t *local;
+	crypt_private_t *priv;
+	struct master_cipher_info *master;
+	struct crypt_inode_info *info;
+
+	priv = this->private;
+	master = get_master_cinfo(priv);
+
+	if (master_alg_atomic(master)) {
+		/*
+		 * We can't open O_WRONLY, because we
+		 * need to do read-modify-write.
+		 */
+		if ((flags & O_ACCMODE) == O_WRONLY)
+			flags = (flags & ~O_ACCMODE) | O_RDWR;
+		/*
+		 * Make sure that out translated offsets
+		 * and counts won't be ignored
+		 */
+		flags &= ~O_APPEND;
+	}
+	local = crypt_alloc_local(frame, this, GF_FOP_CREATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	data = dict_get(xdata, "gfid-req");
+	if (!data) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING, "gfid not found");
+		goto error;
+	}
+	if (data->len != sizeof(uuid_t)) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad gfid size (%d), should be %d",
+		       (int)data->len, (int)sizeof(uuid_t));
+		goto error;
+	}
+	info = alloc_inode_info(local, loc);
+	if (!info){
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * NOTE:
+	 * format has to be created BEFORE
+	 * proceeding to the untrusted server
+	 */
+	ret = alloc_format_create(local);
+	if (ret) {
+		free_inode_info(info);
+		goto error;
+	}
+	init_inode_info_create(info, master, data);
+
+	ret = create_format(local->format,
+			    loc,
+			    info,
+			    master);
+	if (ret) {
+		free_inode_info(info);
+		goto error;		
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		free_inode_info(info);
+		free_format(local);	
+		goto error;
+	}
+	ret = dict_set_static_bin(local->xattr,
+				  CRYPTO_FORMAT_PREFIX,
+				  local->format,
+				  new_format_size());
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	ret = dict_set(local->xattr, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame,
+		   crypt_create_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->create,
+		   loc,
+		   flags,
+		   mode,
+		   umask,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	gf_log("crypt", GF_LOG_WARNING, "can not create file");
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    -1,
+			    ret,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+/*
+ * FIXME: this should depends on the version of format string
+ */
+static int32_t filter_crypt_xattr(dict_t *dict,
+				  char *key, data_t *value, void *data)
+{
+        dict_del(dict, key);
+        return 0;
+}
+
+static int32_t crypt_fsetxattr(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       dict_t *dict,
+			       int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_fsetxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   fd,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * TBD: verify file metadata before wind
+ */
+static int32_t crypt_setxattr(call_frame_t *frame,
+			      xlator_t *this,
+			      loc_t *loc,
+			      dict_t *dict,
+			      int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_setxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   loc,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t linkop_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0 &&
+	    op_errno == ENOENT &&
+	    local->loc->inode->ia_type == IA_IFLNK) {
+		local->op_ret = 0;
+		local->op_errno = 0;
+	}
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * unpin inode on the server
+ */
+static int32_t link_flush(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  inode_t *inode,
+			  struct iatt *buf,
+			  struct iatt *preparent,
+			  struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	link_unwind(frame);
+	return 0;
+}
+
+void link_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	inode_t *inode;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(link,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	inode = local->inode;
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc) {
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(link,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (inode)
+		inode_unref(inode);
+}
+
+void link_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   link_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->link,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+/*
+ * unlink()
+ */
+static int32_t unlink_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *preparent,
+			    struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unlink_unwind(frame);
+	return 0;
+}
+
+void unlink_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(unlink,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(unlink,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+}
+
+void unlink_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   unlink_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->unlink,
+		   local->loc,
+		   local->flags,
+		   local->xdata);
+}
+
+void rename_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(rename,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);	
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	prenewparent = local->prenewparent;
+	postnewparent = local->postnewparent;	
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc){
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(rename,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    prenewparent,
+			    postnewparent,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (prenewparent)
+		GF_FREE(prenewparent);
+	if (postnewparent)
+		GF_FREE(postnewparent);
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t rename_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	rename_unwind(frame);
+	return 0;
+}
+
+static int32_t rename_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *buf,
+			    struct iatt *preoldparent,
+			    struct iatt *postoldparent,
+			    struct iatt *prenewparent,
+			    struct iatt *postnewparent,
+			    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	dict_unref(local->xdata);
+	local->xdata = NULL;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	local->buf = *buf;
+	local->prebuf = *preoldparent;
+	local->postbuf = *postoldparent;
+	if (prenewparent) {
+		local->prenewparent = GF_CALLOC(1, sizeof(*prenewparent),
+						gf_crypt_mt_iatt);
+		if (!local->prenewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->prenewparent = *prenewparent;
+	}
+	if (postnewparent) {
+		local->postnewparent = GF_CALLOC(1, sizeof(*postnewparent),
+						 gf_crypt_mt_iatt);
+		if (!local->postnewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->postnewparent = *postnewparent;
+	}
+	STACK_WIND(frame,
+		   rename_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	rename_unwind(frame);
+	return 0;
+}
+
+void rename_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   rename_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->rename,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+static int32_t __do_linkop(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_wind_handler_t wind_fn;
+	linkop_unwind_handler_t unwind_fn;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret >= 0)
+		wind_fn(frame, this);
+	else {
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+		unwind_fn(frame);
+	}
+	return 0;
+}
+
+static int32_t do_linkop(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if(op_ret < 0)
+		goto error;
+
+	lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:		
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * Update the metadata string (against the new pathname);
+ * submit the result
+ */
+static int32_t linkop_begin(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	gf_boolean_t upload_info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+	struct crypt_inode_info *info;
+	data_t *old_mtd;
+	uint32_t new_mtd_size;
+	uint64_t value = 0;
+	void (*unwind_fn)(call_frame_t *frame);
+	void (*wind_fn)(call_frame_t *frame, xlator_t *this);
+	mtd_op_t mop;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	mop = linkop_mtdop_dispatch(local->fop);
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto wind;
+	if (op_ret < 0)
+		/*
+		 * verification failed
+		 */
+		goto error;
+
+	old_mtd = dict_get(xdata, CRYPTO_FORMAT_PREFIX);
+	if (!old_mtd) {
+		op_errno = EIO;
+		gf_log (this->name, GF_LOG_DEBUG,
+			"Metadata string wasn't found");
+		goto error;
+	}
+	new_mtd_size = format_size(mop, old_mtd->len);
+	op_errno = alloc_format(local, new_mtd_size);
+	if (op_errno)
+		goto error;
+	/*
+	 * check for cached info 
+	 */
+	op_ret = inode_ctx_get(fd->inode, this, &value);
+	if (op_ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log (this->name, GF_LOG_WARNING,
+				"Inode info was not found");
+			op_errno = EINVAL;
+			goto error;
+		}
+		/*
+		 * info was found in the cache
+		 */
+		local->info = info;
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info wasn't found in the cache;
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info)
+			goto error;
+		init_inode_info_head(info, fd);
+		local->info = info;
+		upload_info = _gf_true;
+	}
+	op_errno = open_format((unsigned char *)old_mtd->data,
+			       old_mtd->len,
+			       local->loc,
+			       info,
+			       get_master_cinfo(priv),
+			       local,
+			       upload_info);
+	if (op_errno)
+		goto error;
+	if (upload_info == _gf_true) {
+		op_errno = init_inode_info_tail(info,
+						get_master_cinfo(priv));
+		if (op_errno)
+			goto error;
+		op_errno = inode_ctx_put(fd->inode, this,
+					 (uint64_t)(long)(info));
+		if (op_errno == -1) {
+			op_errno = EIO;
+			goto error;
+		}
+	}
+	/*
+	 * update the format string (append/update/cup a MAC)
+	 */
+	op_errno = update_format(local->format,
+				 (unsigned char *)old_mtd->data,
+				 old_mtd->len,
+				 local->mac_idx,
+				 mop,
+				 local->newloc,
+				 info,
+				 get_master_cinfo(priv),
+				 local);
+	if (op_errno)
+		goto error;
+	/*
+	 * store the new format string on the server
+	 */
+	if (new_mtd_size) {
+		op_errno = dict_set_static_bin(local->xattr,
+					       CRYPTO_FORMAT_PREFIX,
+					       local->format,
+					       new_mtd_size);
+		if (op_errno)
+			goto error;
+	}
+	STACK_WIND(frame,
+		   do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   local->loc,
+		   local->xattr,
+		   0,
+		   NULL);
+	return 0;
+ wind:
+	wind_fn(frame, this);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t linkop_grab_local(call_frame_t *frame,
+				 xlator_t *this,
+				 loc_t *oldloc,
+				 loc_t *newloc,
+				 int flags, dict_t *xdata,
+				 glusterfs_fop_t op)
+{
+	int32_t ret = ENOMEM;
+	fd_t *fd;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, op);
+	if (!local)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	fd = fd_create(oldloc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->flags = flags;
+	local->loc = GF_CALLOC(1, sizeof(*oldloc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, oldloc);
+	if (ret) {
+		GF_FREE(local->loc);
+		local->loc = NULL;
+		goto error;
+	}
+	if (newloc) {
+		local->newloc = GF_CALLOC(1, sizeof(*newloc), gf_crypt_mt_loc);
+		if (!local->newloc) {
+			GF_FREE(local->loc);
+			loc_wipe(local->loc);
+			goto error;
+		}
+		memset(local->newloc, 0, sizeof(*local->newloc));
+		ret = loc_copy(local->newloc, newloc);
+		if (ret) {
+			GF_FREE(local->loc);
+			loc_wipe(local->loc);
+			GF_FREE(local->newloc);
+			goto error;
+		}
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	return 0;
+ error:
+	if (local->xdata)
+		dict_unref(local->xdata);
+	if (local->fd)
+		fd_unref(local->fd);
+	local->fd = 0;
+	local->loc = NULL;
+	local->newloc = NULL;
+
+	local->op_ret = -1;
+	local->op_errno = ret;
+
+	return ret;
+}
+
+/*
+ * read and verify locked metadata against the old pathname (via open);
+ * update the metadata string in accordance with the new pathname;
+ * submit modified metadata;
+ * wind;
+ */
+static int32_t linkop(call_frame_t *frame,
+		      xlator_t *this,
+		      loc_t *oldloc,
+		      loc_t *newloc,
+		      int flags,
+		      dict_t *xdata,
+		      glusterfs_fop_t op)
+{
+	int32_t ret;
+	dict_t *dict;
+	crypt_local_t *local;
+	void (*unwind_fn)(call_frame_t *frame);
+
+	unwind_fn = linkop_unwind_dispatch(op);
+
+	ret = linkop_grab_local(frame, this, oldloc, newloc, flags, xdata, op);
+	local = frame->local;
+	if (ret)
+		goto error;
+	dict = dict_new();
+	if (!dict) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * Set a message to crypt_open() that we need
+	 * locked metadata string.
+	 * All link operations (link, unlink, rename)
+	 * need write lock
+	 */
+	msgflags_set_mtd_wlock(&local->msgflags);
+	ret = dict_set_static_bin(dict,
+				  MSGFLAGS_PREFIX,
+				  &local->msgflags,
+				  sizeof(local->msgflags));
+	if (ret) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not set dict");
+		dict_unref(dict);
+		goto error;
+	}
+	/*
+	 * verify metadata against the old pathname
+	 * and retrieve locked metadata string
+	 */
+	STACK_WIND(frame,
+		   linkop_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   oldloc,
+		   O_RDWR,
+		   local->fd,
+		   dict);
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = ret;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t crypt_link(call_frame_t *frame, xlator_t *this,
+			  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_LINK);
+}
+
+static int32_t crypt_unlink(call_frame_t *frame, xlator_t *this,
+			    loc_t *loc, int flags, dict_t *xdata)
+{
+	return linkop(frame, this, loc, NULL, flags, xdata, GF_FOP_UNLINK);
+}
+
+static int32_t crypt_rename(call_frame_t *frame, xlator_t *this,
+			    loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_RENAME);
+}
+
+static void put_one_call_open(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		fd_t *fd = local->fd;
+		loc_t *loc = local->loc;
+		dict_t *xdata = local->xdata;
+
+		STACK_UNWIND_STRICT(open,
+				    frame,
+				    local->op_ret,
+				    local->op_errno,
+				    fd,
+				    xdata);
+		fd_unref(fd);
+		if (xdata)
+			dict_unref(xdata);
+		loc_wipe(loc);
+		GF_FREE(loc);
+	}
+}
+
+static int32_t __crypt_readv_done(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	/* read deals with data configs only */
+	struct iovec *avec = local->data_conf.avec;
+	char **pool = local->data_conf.pool;
+	int blocks_in_pool = local->data_conf.blocks_in_pool;
+	struct iobref *iobref = local->iobref;
+	struct iobref *iobref_data = local->iobref_data;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "readv unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	dump_plain_text(local, avec);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "readv: ret_to_user: %d, iovec len: %d, ia_size: %llu",
+	       (int)(local->rw_count > 0 ? local->rw_count : local->op_ret),
+	       (int)(local->rw_count > 0 ? iovec_get_size(avec, local->data_conf.acount) : 0),
+	       (unsigned long long)local->buf.ia_size);
+
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    local->rw_count > 0 ? local->rw_count : local->op_ret,
+			    local->op_errno,
+			    avec,
+			    avec ? local->data_conf.acount : 0,
+			    &local->buf,
+			    local->iobref,
+			    local_xdata);
+
+	free_avec(avec, pool, blocks_in_pool);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobref)
+		iobref_unref(iobref);
+	if (iobref_data)
+		iobref_unref(iobref_data);
+	return 0;
+}
+
+static void crypt_readv_done(call_frame_t *frame, xlator_t *this)
+{
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 *  don't unlock (it will be done by the parent)
+		 */
+		__crypt_readv_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		crypt_local_t *local = frame->local;
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_readv_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+}
+
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local))
+		crypt_readv_done(frame, this);
+}
+
+static int32_t __crypt_writev_done(call_frame_t *frame,
+				   void *cookie,
+				   xlator_t *this,
+				   int32_t op_ret,
+				   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	int32_t ret_to_user;
+
+	if (local->xattr)
+		dict_unref(local->xattr);
+	/*
+	 * Calculate amout of butes to be returned
+	 * to user. We need to subtract paddings that
+	 * have been written as a part of atom.
+	 */
+	/*
+	 * subtract head padding
+	 */
+	if (local->rw_count == 0)
+		/*
+		 * Nothing has been written, it must be an error
+		 */
+		ret_to_user = local->op_ret;
+	else if (local->rw_count <= local->data_conf.off_in_head) {
+		gf_log("crypt", GF_LOG_WARNING, "Incomplete write");
+		ret_to_user = 0;
+	}
+	else
+		ret_to_user = local->rw_count -
+			local->data_conf.off_in_head;
+	/*
+	 * subtract tail padding
+	 */
+	if (ret_to_user > local->data_conf.orig_size)
+		ret_to_user = local->data_conf.orig_size;
+
+	if (local->iobref)
+		iobref_unref(local->iobref);
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "writev: ret_to_user: %d", ret_to_user);
+
+	STACK_UNWIND_STRICT(writev,
+			    frame,
+			    ret_to_user,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_writev_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 * don't unlock (it will be done by the parent)
+		 */
+		__crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_writev_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+	return 0;
+}
+
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_writev_done(frame, NULL,
+						  this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_writev_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+static int32_t __crypt_ftruncate_done(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	char *iobase = local->vec.iov_base;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "ftruncate unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "ftruncate, return to user: presize=%llu, postsize=%llu",
+	       (unsigned long long)local->prebuf.ia_size,
+	       (unsigned long long)local->postbuf.ia_size);
+
+	STACK_UNWIND_STRICT(ftruncate,
+			    frame,
+			    local->op_ret < 0 ? -1 : 0,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobase)
+		GF_FREE(iobase);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_done(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct gf_flock  lock  = {0, };
+
+	dict_unref(local->xattr);
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	lock.l_type   = F_UNLCK;
+	lock.l_whence = SEEK_SET;
+	lock.l_start  = 0;
+	lock.l_len    = 0;
+	lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __crypt_ftruncate_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+}
+
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_ftruncate_done(frame, NULL,
+						     this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_ftruncate_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_ftruncate_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+/*
+ * load regular file size for some FOPs
+ */
+static int32_t load_file_size(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      dict_t *dict,
+			      dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0)
+		goto unwind;
+	/*
+	 * load regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		if (local->xdata)
+			dict_unref(local->xdata);
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto unwind;
+	}
+	local->buf.ia_size = data_to_uint64(data);
+
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "FOP %d: Translate regular file to %llu",
+	       local->fop,
+	       (unsigned long long)local->buf.ia_size);
+ unwind:	
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_LOOKUP:
+		STACK_UNWIND_STRICT(lookup,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? local->inode : NULL,
+				    op_ret >= 0 ? &local->buf : NULL,				    
+				    local->xdata,
+				    op_ret >= 0 ? &local->postbuf : NULL);
+		break;
+	case GF_FOP_READ:
+		STACK_UNWIND_STRICT(readv,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    NULL,
+				    0,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    NULL,
+				    NULL);
+		break;
+	default:
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Improper file operation %d", local->fop);
+	}
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (local_inode)
+		inode_unref(local_inode);
+	return 0;
+}
+
+static int32_t crypt_stat_common_cbk(call_frame_t *frame,
+				     void *cookie,
+				     xlator_t *this,
+				     int32_t op_ret,
+				     int32_t op_errno,
+				     struct iatt *buf, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->buf = *buf;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->fgetxattr,
+			   local->fd,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->getxattr,
+			   local->loc,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+ unwind:
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+}
+
+static int32_t crypt_fstat(call_frame_t *frame,
+			   xlator_t *this,
+			   fd_t *fd, dict_t *xdata)
+{
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FSTAT);
+	if (!local)
+		goto error;
+	local->fd = fd_ref(fd);
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(fstat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_stat(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_STAT);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->stat,
+		   loc,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(stat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_lookup_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				inode_t *inode,
+				struct iatt *buf, dict_t *xdata,
+				struct iatt *postparent)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->postbuf = *postparent;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	uuid_copy(local->loc->gfid, buf->ia_gfid);
+
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ unwind:
+	loc_wipe(local->loc);
+	GF_FREE(local->loc);
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    inode,
+			    buf,
+			    xdata,
+			    postparent);
+	return 0;
+}
+
+static int32_t crypt_lookup(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_LOOKUP);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "Lookup %s", loc->path);
+	STACK_WIND(frame,
+		   crypt_lookup_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->lookup,
+		   loc,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * for every regular directory entry find its real file size
+ * and update stat's buf properly
+ */
+static int32_t crypt_readdirp_cbk(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno,
+				  gf_dirent_t *entries, dict_t *xdata)
+{
+	gf_dirent_t *entry = NULL;
+
+	if (op_ret < 0)
+		goto unwind;
+
+	list_for_each_entry (entry, (&entries->list), list) {
+		data_t *data;
+
+		if (!IA_ISREG(entry->d_stat.ia_type))
+			continue;
+		data = dict_get(entry->dict, FSIZE_XATTR_PREFIX);
+		if (!data){
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size of direntry not found");
+			op_errno = EIO;
+			op_ret = -1;
+			break;
+		}
+		entry->d_stat.ia_size = data_to_uint64(data);
+	}
+ unwind:
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+/*
+ * ->readdirp() fills in-core inodes, so we need to set proper
+ * file sizes for all directory entries of the parent @fd.
+ * Actual updates take place in ->crypt_readdirp_cbk()
+ */
+static int32_t crypt_readdirp(call_frame_t *frame, xlator_t *this,
+			      fd_t *fd, size_t size, off_t offset,
+			      dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+
+	if (!xdata) {
+		xdata = dict_new();
+		if (!xdata)
+			goto error;
+	}
+	else
+		dict_ref(xdata);
+	/*
+	 * make sure that we'll have real file sizes at ->readdirp_cbk()
+	 */
+	ret = dict_set(xdata, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(xdata);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_readdirp_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp,
+		   fd,
+		   size,
+		   offset,
+		   xdata);
+	dict_unref(xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readdirp, frame, -1, ret, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_access(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t mask, dict_t *xdata)
+{
+	gf_log(this->name, GF_LOG_WARNING,
+	       "NFS mounts of encrypted volumes are unsupported");
+	STACK_UNWIND_STRICT(access, frame, -1, EPERM, NULL);
+	return 0;
+}
+
+int32_t master_set_block_size (xlator_t *this, crypt_private_t *priv,
+			       dict_t *options)
+{
+	uint64_t block_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("block-size", block_size, options,
+				 size, error);
+	else
+		GF_OPTION_INIT("block-size", block_size, size, error);
+
+	switch (block_size) {
+	case 512:
+		master->m_block_bits = 9;
+		break;
+	case 1024:
+		master->m_block_bits = 10;
+		break;
+	case 2048:
+		master->m_block_bits = 11;
+		break;
+	case 4096:
+		master->m_block_bits = 12;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: unsupported block size %llu",
+		       (unsigned long long)block_size);
+		goto error;
+	}
+	return 0;
+ error:
+	return -1;
+}
+
+int32_t master_set_alg(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_alg = AES_CIPHER_ALG;
+	return 0;
+}
+
+int32_t master_set_mode(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_mode = XTS_CIPHER_MODE;
+	return 0;
+}
+
+/*
+ * set key size in bits to the master info
+ * Pre-conditions: cipher mode in the master info is uptodate.
+ */
+static int master_set_data_key_size (xlator_t *this, crypt_private_t *priv,
+				     dict_t *options)
+{
+	int32_t ret;
+	uint64_t key_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("data-key-size", key_size, options,
+				 size, error);
+	else
+		GF_OPTION_INIT("data-key-size", key_size, size, error);
+
+	ret = data_cipher_algs[master->m_alg][master->m_mode].check_key(key_size);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: wrong bin key size %llu for alg %d mode %d",
+		       (unsigned long long)key_size,
+		       (int)master->m_alg,
+		       (int)master->m_mode);
+		goto error;
+	}
+	master->m_dkey_size = key_size;
+	return 0;
+ error:
+	return -1;
+}
+
+static int is_hex(char *s) {
+	return  ('0' <= *s && *s <= '9') || ('a' <= *s && *s <= 'f');
+}
+
+static int parse_hex_buf(xlator_t *this, char *src, unsigned char *dst,
+			 int hex_size)
+{
+	int i;
+	int hex_byte = 0;
+
+	for (i = 0; i < (hex_size / 2); i++) {
+		if (!is_hex(src + i*2) || !is_hex(src + i*2 + 1)) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: not hex symbol in key");
+			return -1;
+		}
+		if (sscanf(src + i*2, "%2x", &hex_byte) != 1) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: can not parse hex key");
+			return -1;
+		}
+		dst[i] = hex_byte & 0xff;
+	}
+	return 0;
+}
+
+/*
+ * Parse options;
+ * install master volume key
+ */
+int32_t master_set_master_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	int32_t ret;
+	FILE *file = NULL;
+
+	int32_t key_size;
+	char *opt_key_file_pathname = NULL;
+
+	unsigned char bin_buf[MASTER_VOL_KEY_SIZE];
+	char hex_buf[2 * MASTER_VOL_KEY_SIZE];
+
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	/*
+	 * extract master key passed via option
+	 */
+	GF_OPTION_INIT("master-key", opt_key_file_pathname, path, bad_key);
+
+	if (!opt_key_file_pathname) {
+		gf_log(this->name, GF_LOG_ERROR, "FATAL: missing master key");
+		return -1;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "handling file key %s",
+	       opt_key_file_pathname);
+
+	file = fopen(opt_key_file_pathname, "r");
+	if (file == NULL) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: can not open file with master key");
+		return -1;
+	}
+	/*
+	 * extract hex key
+	 */
+	key_size = fread(hex_buf, 1, sizeof(hex_buf), file);
+	if (key_size < sizeof(hex_buf)) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: master key is too short");
+		goto bad_key;
+	}
+	ret = parse_hex_buf(this, hex_buf, bin_buf, key_size);
+	if (ret)
+		goto bad_key;
+	memcpy(master->m_key, bin_buf, MASTER_VOL_KEY_SIZE);
+	memset(hex_buf, 0, sizeof(hex_buf));
+	fclose(file);
+
+ 	memset(bin_buf, 0, sizeof(bin_buf));
+	return 0;
+ bad_key:
+	gf_log(this->name, GF_LOG_ERROR, "FATAL: bad master key");
+	if (file)
+		fclose(file);
+	memset(bin_buf, 0, sizeof(bin_buf));
+	return -1;
+}
+
+/*
+ * Derive volume key for object-id authentication
+ */
+int32_t master_set_nmtd_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	return get_nmtd_vol_key(get_master_cinfo(priv));
+}
+
+int32_t crypt_init_xlator(xlator_t *this)
+{
+	int32_t ret;
+	crypt_private_t *priv = this->private; 
+
+	ret = master_set_alg(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_mode(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_block_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_data_key_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_master_vol_key(this, priv);
+	if (ret)
+		return ret;
+	return master_set_nmtd_vol_key(this, priv);
+}
+
+static int32_t crypt_alloc_private(xlator_t *this)
+{
+	this->private = GF_CALLOC(1, sizeof(crypt_private_t), gf_crypt_mt_priv);
+	if (!this->private) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Can not allocate memory for private data");
+		return ENOMEM;
+	}
+	return 0;
+}
+
+static void crypt_free_private(xlator_t *this)
+{
+	crypt_private_t *priv = this->private;
+	if (priv) {
+		memset(priv, 0, sizeof(*priv));
+		GF_FREE(priv);
+	}
+}
+
+int32_t reconfigure (xlator_t *this, dict_t *options)
+{
+	int32_t ret = -1;
+	crypt_private_t *priv = NULL;
+
+	GF_VALIDATE_OR_GOTO ("crypt", this, error);
+	GF_VALIDATE_OR_GOTO (this->name, this->private, error);
+	GF_VALIDATE_OR_GOTO (this->name, options, error);
+
+	priv = this->private;
+
+	ret = master_set_block_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure block size");
+		goto error;
+	}
+	ret = master_set_data_key_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure data key size");
+		goto error;
+	}
+	return 0;
+ error:
+	return ret;
+}
+
+int32_t init(xlator_t *this)
+{
+	int32_t ret;
+
+	if (!this->children || this->children->next) {
+		gf_log ("crypt", GF_LOG_ERROR,
+			"FATAL: crypt should have exactly one child");
+		return EINVAL;
+	}
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+	ret = crypt_alloc_private(this);
+	if (ret)
+		return ret;
+	ret = crypt_init_xlator(this);
+	if (ret)
+		goto error;
+	this->local_pool = mem_pool_new(crypt_local_t, 64);
+        if (!this->local_pool) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "failed to create local_t's memory pool");
+		ret = ENOMEM;
+                goto error;
+        }
+	gf_log ("crypt", GF_LOG_INFO, "crypt xlator loaded");
+	return 0;
+ error:
+	crypt_free_private(this);
+	return ret;
+}
+
+void fini (xlator_t *this)
+{
+	crypt_free_private(this);
+}
+
+struct xlator_fops fops = {
+	.readv        = crypt_readv,
+	.writev       = crypt_writev,
+	.truncate     = crypt_truncate,
+	.ftruncate    = crypt_ftruncate,
+	.setxattr     = crypt_setxattr,
+	.fsetxattr    = crypt_fsetxattr,
+	.link         = crypt_link,
+	.unlink       = crypt_unlink,
+	.rename       = crypt_rename,
+	.open         = crypt_open,
+	.create       = crypt_create,
+	.stat         = crypt_stat,
+	.fstat        = crypt_fstat,
+	.lookup       = crypt_lookup,
+	.readdirp     = crypt_readdirp,
+	.access       = crypt_access   
+};
+
+struct xlator_cbks cbks = {
+	.forget       = crypt_forget
+};
+
+struct volume_options options[] = {
+	{ .key = {"master-key"},
+	  .type = GF_OPTION_TYPE_PATH,
+	  .description = "Pathname of regular file which contains master volume key"
+	},
+	{ .key = {"data-key-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Data key size (bits)",
+	  .min = 256,
+	  .max = 512,
+	  .default_value = "256",
+	},
+	{ .key = {"block-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Atom size (bits)",
+	  .min = 512,
+	  .max = 4096,
+	  .default_value = "4096"
+	},
+	{ .key  = {NULL} },
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt.h b/xlators/encryption/crypt/src/crypt.h
new file mode 100644
index 000000000..01a8542ab
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.h
@@ -0,0 +1,899 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_H__
+#define __CRYPT_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <openssl/aes.h>
+#include <openssl/evp.h>
+#include <openssl/sha.h>
+#include <openssl/hmac.h>
+#include <openssl/cmac.h>
+#include <openssl/modes.h>
+#include "crypt-mem-types.h"
+
+#define CRYPT_XLATOR_ID  (0)
+
+#define MAX_IOVEC_BITS (3)
+#define MAX_IOVEC (1 << MAX_IOVEC_BITS)
+#define KEY_FACTOR_BITS  (6)
+
+#define DEBUG_CRYPT (0)
+#define TRIVIAL_TFM (0)
+
+#define CRYPT_MIN_BLOCK_BITS (9)
+#define CRYPT_MAX_BLOCK_BITS (12)
+
+#define MASTER_VOL_KEY_SIZE (32)
+#define NMTD_VOL_KEY_SIZE (16)
+
+struct crypt_key {
+	uint32_t len;
+	const char *label;	
+};
+
+/*
+ * Add new key types to the end of this
+ * enumeration but before LAST_KEY_TYPE
+ */
+typedef enum {
+	MASTER_VOL_KEY,
+	NMTD_VOL_KEY,
+	NMTD_LINK_KEY,
+	EMTD_FILE_KEY,
+	DATA_FILE_KEY_256,
+	DATA_FILE_KEY_512,
+	LAST_KEY_TYPE
+}crypt_key_type;
+
+struct kderive_context {
+	const unsigned char *pkey;/* parent key */
+	uint32_t pkey_len;        /* parent key size, bits */
+	uint32_t ckey_len;        /* child key size, bits */
+	unsigned char *fid;       /* fixed input data, NIST 800-108, 5.1 */
+	uint32_t fid_len;         /* fid len, bytes */
+	unsigned char *out;       /* contains child keying material */
+	uint32_t out_len;         /* out len, bytes */
+};
+
+typedef enum {
+	DATA_ATOM,
+	HOLE_ATOM,
+	LAST_DATA_TYPE
+}atom_data_type;
+
+typedef enum {
+	HEAD_ATOM,
+	TAIL_ATOM,
+	FULL_ATOM,
+	LAST_LOCALITY_TYPE
+}atom_locality_type;
+
+typedef enum {
+	MTD_CREATE,
+	MTD_APPEND,
+	MTD_OVERWRITE,
+	MTD_CUT,
+	MTD_LAST_OP
+} mtd_op_t;
+
+struct xts128_context {
+	void      *key1, *key2;
+	block128_f block1,block2;
+};
+
+struct object_cipher_info {
+	cipher_alg_t  o_alg;
+        cipher_mode_t o_mode;
+	uint32_t      o_block_bits;
+        uint32_t      o_dkey_size; /* raw data key size in bits */
+	union {
+		struct {
+			unsigned char ivec[16];
+			AES_KEY dkey[2];
+			AES_KEY tkey;    /* key used for tweaking */
+			XTS128_CONTEXT xts;
+		} aes_xts;
+	} u;
+};
+
+struct master_cipher_info {
+	/*
+	 * attributes inherited by newly created regular files
+	 */
+	cipher_alg_t  m_alg;
+        cipher_mode_t m_mode;
+	uint32_t      m_block_bits;
+        uint32_t      m_dkey_size; /* raw key size in bits */
+	/*
+	 * master key
+	 */
+	unsigned char m_key[MASTER_VOL_KEY_SIZE];
+	/* 
+	 * volume key for oid authentication
+	 */
+	unsigned char m_nmtd_key[NMTD_VOL_KEY_SIZE];
+};
+
+/*
+* This info is not changed during file's life
+ */
+struct crypt_inode_info {
+#if DEBUG_CRYPT
+	loc_t *loc; /* pathname that the file has been
+		       opened, or created with */
+#endif
+	uint16_t nr_minor;
+	uuid_t oid;
+	struct object_cipher_info cinfo;
+};
+
+/*
+ * this should locate in secure memory
+ */
+typedef struct {
+	struct master_cipher_info master;
+} crypt_private_t;
+
+static inline struct master_cipher_info *get_master_cinfo(crypt_private_t *priv)
+{
+	return &priv->master;
+}
+
+static inline struct object_cipher_info *get_object_cinfo(struct crypt_inode_info
+							 *info)
+{
+	return &info->cinfo;
+}
+
+/*
+ * this describes layouts and properties
+ * of atoms in an aligned vector
+ */
+struct avec_config {
+	uint32_t atom_size;
+	atom_data_type type;
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head;
+	uint32_t off_in_tail;
+	uint32_t gap_in_tail;
+	uint32_t nr_full_blocks;
+
+	struct iovec *avec; /* aligned vector */
+	uint32_t acount; /* number of avec components. The same
+			  * as number of occupied logical blocks */
+	char **pool;
+	uint32_t blocks_in_pool;
+	uint32_t cursor; /* makes sense only for ordered writes,
+			  * so there is no races on this counter.
+			  *
+			  * Cursor is per-config object, we don't
+			  * reset cursor for atoms of different
+			  * localities (head, tail, full)
+			  */
+};
+
+
+typedef struct {
+	glusterfs_fop_t fop; /* code of FOP this local info built for */
+	fd_t *fd;
+	inode_t *inode;
+	loc_t *loc;
+	int32_t mac_idx;
+	loc_t *newloc;
+	int32_t flags;
+	int32_t wbflags;
+	struct crypt_inode_info *info;
+	struct iobref *iobref;
+	struct iobref *iobref_data;
+	off_t offset;
+
+	uint64_t old_file_size; /* per FOP, retrieved under lock held */
+	uint64_t cur_file_size; /* per iteration, before issuing IOs */
+	uint64_t new_file_size; /* per iteration, after issuing IOs */
+
+	uint64_t io_offset; /* offset of IOs issued per iteration */
+	uint64_t io_offset_nopad; /* offset of user's data in the atom */
+	uint32_t io_size; /* size of IOs issued per iteration */
+	uint32_t io_size_nopad; /* size of user's data in the IOs */
+	uint32_t eof_padding_size; /* size od EOF padding in the IOs */
+
+	gf_lock_t call_lock; /* protect nr_calls from many cbks */
+	int32_t nr_calls;
+
+	atom_data_type active_setup; /* which setup (hole or date)
+					is currently active */
+	/* data setup */
+	struct avec_config data_conf;
+
+	/* hole setup */
+	int hole_conv_in_proggress;
+	gf_lock_t hole_lock; /* protect hole config from many cbks */
+	int hole_handled;
+	struct avec_config hole_conf;
+	struct iatt buf;
+	struct iatt prebuf;
+	struct iatt postbuf;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+	int32_t op_ret;
+	int32_t op_errno;
+	int32_t rw_count; /* total read or written */
+	gf_lock_t rw_count_lock; /* protect the counter above */
+	unsigned char *format; /* for create, update format string */
+	uint32_t format_size;
+	uint32_t msgflags; /* messages for crypt_open() */
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iovec vec; /* contains last file's atom for
+			     read-prune-write sequence */
+	gf_boolean_t custom_mtd;
+	/*
+	 * the next 3 fields are used by readdir and friends
+	 */
+	gf_dirent_t *de; /* directory entry */
+	char *de_path; /* pathname of directory entry */
+	uint32_t de_prefix_len; /* lenght of the parent's pathname */
+	gf_dirent_t *entries;
+
+	uint32_t update_disk_file_size:1;
+} crypt_local_t;
+
+/* This represents a (read)modify-write atom */
+struct rmw_atom {
+	atom_locality_type locality;
+	/*
+	 * read-modify-write sequence of the atom
+	 */
+	int32_t (*rmw)(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iovec *vec,
+		       int32_t count,
+		       struct iatt *stbuf,
+		       struct iobref *iobref,
+		       dict_t *xdata);
+	/*
+	 * offset of the logical block in a file
+	 */
+	loff_t (*offset_at)(call_frame_t *frame,
+			    struct object_cipher_info *object);
+	/*
+	 * IO offset in an atom
+	 */
+	uint32_t (*offset_in)(call_frame_t *frame,
+			      struct object_cipher_info *object);
+	/*
+	 * number of bytes of plain text of this atom that user
+	 * wants to read/write.
+	 * It can be smaller than atom_size in the case of head
+	 * or tail atoms.
+	 */
+	uint32_t (*io_size_nopad)(call_frame_t *frame,
+				  struct object_cipher_info *object);
+	/*
+	 * which iovec represents the atom
+	 */
+	struct iovec *(*get_iovec)(call_frame_t *frame, uint32_t count);
+	/*
+	 * how many bytes of partial block should be uptodated by
+	 * reading from disk.
+	 * This is used to perform a read component of RMW (read-modify-write).
+	 */
+	uint32_t (*count_to_uptodate)(call_frame_t *frame, struct object_cipher_info *object);
+	struct avec_config *(*get_config)(call_frame_t *frame);
+};
+
+struct data_cipher_alg {
+	gf_boolean_t atomic; /* true means that algorithm requires
+				to pad data before cipher transform */
+	gf_boolean_t should_pad; /* true means that algorithm requires
+				    to pad the end of file with extra-data */
+	uint32_t blkbits; /* blksize = 1 << blkbits */
+	/*
+	 * any preliminary sanity checks goes here
+	 */
+	int32_t (*init)(void);
+	/*
+	 * set alg-mode specific inode info
+	 */
+	int32_t (*set_private)(struct crypt_inode_info *info,
+			       struct master_cipher_info *master);
+	/*
+	 * check alg-mode specific data key
+	 */
+	int32_t (*check_key)(uint32_t key_size);
+	void (*set_iv)(off_t offset, struct object_cipher_info *object);
+	int32_t (*encrypt)(const unsigned char *from, unsigned char *to,
+			   size_t length, off_t offset, const int enc,
+			   struct object_cipher_info *object);
+};
+
+/*
+ * version-dependent metadata loader
+ */
+struct crypt_mtd_loader {
+	/*
+	 * return core format size
+	 */
+	size_t (*format_size)(mtd_op_t op, size_t old_size);
+	/*
+	 * pack version-specific metadata of an object
+	 * at ->create()
+	 */
+	int32_t (*create_format)(unsigned char *wire,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master);
+	/*
+	 * extract version-specific metadata of an object
+	 * at ->open() time
+	 */
+	int32_t (*open_format)(unsigned char *wire,
+			       int32_t len,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local,
+			       gf_boolean_t load_info);
+	int32_t (*update_format)(unsigned char *new,
+				 unsigned char *old,
+				 size_t old_len,
+				 int32_t mac_idx,
+				 mtd_op_t op,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master,
+				 crypt_local_t *local);
+};
+
+typedef int32_t (*end_writeback_handler_t)(call_frame_t *frame,
+					   void *cookie,
+					   xlator_t *this,
+					   int32_t op_ret,
+					   int32_t op_errno,
+					   struct iatt *prebuf,
+					   struct iatt *postbuf,
+					   dict_t *xdata);
+typedef void (*linkop_wind_handler_t)(call_frame_t *frame, xlator_t *this);
+typedef void (*linkop_unwind_handler_t)(call_frame_t *frame);
+
+
+/* Declarations */
+
+/* keys.c */
+extern struct crypt_key crypt_keys[LAST_KEY_TYPE];
+int32_t get_nmtd_vol_key(struct master_cipher_info *master);
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key);
+/* data.c */
+extern struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE];
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf);
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count);
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop);
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype);
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t setup_gap_in_tail);
+
+/* metadata.c */
+extern struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER];
+
+int32_t alloc_format(crypt_local_t *local, size_t size);
+int32_t alloc_format_create(crypt_local_t *local);
+void free_format(crypt_local_t *local);
+size_t format_size(mtd_op_t op, size_t old_size);
+size_t new_format_size(void);
+int32_t open_format(unsigned char *str, int32_t len, loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master, crypt_local_t *local,
+		    gf_boolean_t load_info);
+int32_t update_format(unsigned char *new, unsigned char *old,
+		      size_t old_len, int32_t mac_idx, mtd_op_t op, loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local);
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master);
+
+/* atom.c */
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality);
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype);
+void submit_full(call_frame_t *frame, xlator_t *this);
+
+/* crypt.c */
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop);
+static size_t iovec_get_size(struct iovec *vec, uint32_t count);
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom, off_t io_offset,
+				uint32_t io_size);
+void link_wind(call_frame_t *frame, xlator_t *this);
+void unlink_wind(call_frame_t *frame, xlator_t *this);
+void link_unwind(call_frame_t *frame);
+void unlink_unwind(call_frame_t *frame);
+void rename_wind(call_frame_t *frame, xlator_t *this);
+void rename_unwind(call_frame_t *frame);
+
+/* Inline functions */
+
+static inline size_t iovec_get_size(struct iovec *vec, uint32_t count)
+{
+	int i;
+	size_t size = 0;
+	for (i = 0; i < count; i++)
+		size += vec[i].iov_len;
+	return size;
+}
+
+static inline int32_t crypt_xlator_id(void)
+{
+	return CRYPT_XLATOR_ID;
+}
+
+static inline mtd_loader_id current_mtd_loader(void)
+{
+	return MTD_LOADER_V1;
+}
+
+static inline uint32_t master_key_size (void)
+{
+	return crypt_keys[MASTER_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t nmtd_vol_key_size (void)
+{
+	return crypt_keys[NMTD_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t alg_mode_blkbits(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].blkbits;
+}
+
+static inline uint32_t alg_mode_blksize(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return 1 << alg_mode_blkbits(alg, mode);
+}
+
+static inline gf_boolean_t alg_mode_atomic(cipher_alg_t alg,
+					   cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].atomic;
+}
+
+static inline gf_boolean_t alg_mode_should_pad(cipher_alg_t alg,
+					       cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].should_pad;
+}
+
+static inline uint32_t master_alg_blksize(struct master_cipher_info *mr)
+{
+	return alg_mode_blksize(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t master_alg_blkbits(struct master_cipher_info *mr)
+{
+	return alg_mode_blkbits(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_atomic(struct master_cipher_info *mr)
+{
+	return alg_mode_atomic(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_should_pad(struct master_cipher_info *mr)
+{
+	return alg_mode_should_pad(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t object_alg_blksize(struct object_cipher_info *ob)
+{
+	return alg_mode_blksize(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t object_alg_blkbits(struct object_cipher_info *ob)
+{
+	return alg_mode_blkbits(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_atomic(struct object_cipher_info *ob)
+{
+	return alg_mode_atomic(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_should_pad(struct object_cipher_info *ob)
+{
+	return alg_mode_should_pad(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t aes_raw_key_size(struct master_cipher_info *master)
+{
+	return master->m_dkey_size >> 3;
+}
+
+static inline struct avec_config *get_hole_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->hole_conf);
+}
+
+static inline struct avec_config *get_data_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->data_conf);
+}
+
+static inline int32_t get_atom_bits (struct object_cipher_info *object)
+{
+	return object->o_block_bits;
+}
+
+static inline int32_t get_atom_size (struct object_cipher_info *object)
+{
+	return 1 << get_atom_bits(object);
+}
+
+static inline int32_t has_head_block(struct avec_config *conf)
+{
+	return conf->off_in_head ||
+		(conf->acount == 1 && conf->off_in_tail);
+}
+
+static inline int32_t has_tail_block(struct avec_config *conf)
+{
+	return conf->off_in_tail && conf->acount > 1;
+}
+
+static inline int32_t has_full_blocks(struct avec_config *conf)
+{
+	return conf->nr_full_blocks;
+}
+
+static inline int32_t should_submit_head_block(struct avec_config *conf)
+{
+	return has_head_block(conf) && (conf->cursor == 0);
+}
+
+static inline int32_t should_submit_tail_block(struct avec_config *conf)
+{
+	return has_tail_block(conf) && (conf->cursor == conf->acount - 1);
+}
+
+static inline int32_t should_submit_full_block(struct avec_config *conf)
+{
+	uint32_t start = has_head_block(conf) ? 1 : 0;
+
+	return has_full_blocks(conf) &&
+		conf->cursor >= start &&
+		conf->cursor < start + conf->nr_full_blocks;
+}
+
+#if DEBUG_CRYPT
+static inline void crypt_check_input_len(size_t len,
+					 struct object_cipher_info *object)
+{
+	if (object_alg_should_pad(object) && (len & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG, "bad input len: %d", (int)len);
+}
+
+static inline void check_head_block(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a head atom");
+}
+
+static inline void check_tail_block(struct avec_config *conf)
+{
+	if (!has_tail_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a tail atom");
+}
+
+static inline void check_full_block(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a full atom");
+}
+
+static inline void check_cursor_head(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of head atom method");
+	else if (conf->cursor != 0)
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor (%d) is not at head atom",
+		       conf->cursor);
+}
+
+static inline void check_cursor_full(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of full atom method");
+	if (has_head_block(conf) && (conf->cursor == 0))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor is not at full atom");
+}
+
+/*
+ * FIXME: use avec->iov_len to check setup
+ */
+static inline int data_local_invariant(crypt_local_t *local)
+{
+	return 0;
+}
+
+#else
+#define crypt_check_input_len(len, object) noop
+#define check_head_block(conf) noop
+#define check_tail_block(conf) noop
+#define check_full_block(conf) noop
+#define check_cursor_head(conf) noop
+#define check_cursor_full(conf) noop
+
+#endif /* DEBUG_CRYPT */
+
+static inline struct avec_config *conf_by_type(call_frame_t *frame,
+					       atom_data_type dtype)
+{
+	struct avec_config *conf = NULL;
+
+	switch (dtype) {
+	case HOLE_ATOM:
+		conf = get_hole_conf(frame);
+		break;
+	case DATA_ATOM:
+		conf = get_data_conf(frame);
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom type");
+	}
+	return conf;
+}
+
+static inline uint32_t nr_calls_head(struct avec_config *conf)
+{
+	return has_head_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_tail(struct avec_config *conf)
+{
+	return has_tail_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_full(struct avec_config *conf)
+{
+	switch(conf->type) {
+	case HOLE_ATOM:
+		return has_full_blocks(conf);
+	case DATA_ATOM:
+		return has_full_blocks(conf) ?
+			logical_blocks_occupied(0,
+						conf->nr_full_blocks,
+						MAX_IOVEC_BITS)	: 0;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom data type");
+		return 0;
+	}
+}
+
+static inline uint32_t nr_calls(struct avec_config *conf)
+{
+	return nr_calls_head(conf) + nr_calls_tail(conf) + nr_calls_full(conf);
+}
+
+static inline uint32_t nr_calls_data(call_frame_t *frame)
+{
+	return nr_calls(get_data_conf(frame));
+}
+
+static inline uint32_t nr_calls_hole(call_frame_t *frame)
+{
+	return nr_calls(get_hole_conf(frame));
+}
+
+static inline void get_one_call_nolock(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	++local->nr_calls;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", 1);
+}
+
+static inline void get_one_call(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_one_call_nolock(frame);
+	UNLOCK(&local->call_lock);
+}
+
+static inline void get_nr_calls_nolock(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	local->nr_calls += nr;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", nr);
+}
+
+static inline void get_nr_calls(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_nr_calls_nolock(frame, nr);
+	UNLOCK(&local->call_lock);
+}
+
+static inline int put_one_call(crypt_local_t *local)
+{
+	uint32_t last = 0;
+
+	LOCK(&local->call_lock);
+	if (--local->nr_calls == 0)
+		last = 1;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "put %d calls", 1);
+
+	UNLOCK(&local->call_lock);
+	return last;
+}
+
+static inline int is_appended_write(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->orig_offset + conf->orig_size > local->old_file_size;
+}
+
+static inline int is_ordered_mode(call_frame_t *frame)
+{
+#if 0
+	crypt_local_t *local = frame->local;
+	return local->fop == GF_FOP_FTRUNCATE ||
+		(local->fop == GF_FOP_WRITE && is_appended_write(frame));
+#endif
+	return 1;
+}
+
+static inline int32_t hole_conv_completed(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+	return conf->cursor == conf->acount;
+}
+
+static inline int32_t data_write_in_progress(crypt_local_t *local)
+{
+	return local->active_setup == DATA_ATOM;
+}
+
+static inline int32_t parent_is_crypt_xlator(call_frame_t *frame,
+					     xlator_t *this)
+{
+	return frame->parent->this == this;
+}
+
+static inline linkop_wind_handler_t linkop_wind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_wind;
+	case GF_FOP_UNLINK:
+		return unlink_wind;
+	case GF_FOP_RENAME:
+		return rename_wind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;
+	}
+}
+
+static inline linkop_unwind_handler_t linkop_unwind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_unwind;
+	case GF_FOP_UNLINK:
+		return unlink_unwind;
+	case GF_FOP_RENAME:
+		return rename_unwind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;		
+	}	
+}
+
+static inline mtd_op_t linkop_mtdop_dispatch(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_LINK:
+		return MTD_APPEND;
+	case GF_FOP_UNLINK:
+		return MTD_CUT;
+	case GF_FOP_RENAME:
+		return MTD_OVERWRITE;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad link operation %d", fop);
+		return MTD_LAST_OP;
+	}
+}
+
+#endif /* __CRYPT_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/data.c b/xlators/encryption/crypt/src/data.c
new file mode 100644
index 000000000..762fa554a
--- /dev/null
+++ b/xlators/encryption/crypt/src/data.c
@@ -0,0 +1,769 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void set_iv_aes_xts(off_t offset, struct object_cipher_info *object)
+{
+	unsigned char *ivec;
+
+	ivec = object->u.aes_xts.ivec;
+
+	/* convert the tweak into a little-endian byte
+	 * array (IEEE P1619/D16, May 2007, section 5.1)
+	 */
+
+	*((uint64_t *)ivec) = htole64(offset);
+
+	/* ivec is padded with zeroes */
+}
+
+static int32_t aes_set_keys_common(unsigned char *raw_key, uint32_t key_size,
+				   AES_KEY *keys)
+{
+	int32_t ret;
+
+	ret = AES_set_encrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_ENCRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set encrypt key failed");
+		return ret;
+	}
+	ret = AES_set_decrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_DECRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set decrypt key failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * set private cipher info for xts mode
+ */
+static int32_t set_private_aes_xts(struct crypt_inode_info *info,
+				   struct master_cipher_info *master)
+{
+	int ret;
+	struct object_cipher_info *object = get_object_cinfo(info);
+	unsigned char *data_key;
+	uint32_t subkey_size;
+
+	/* init tweak value */
+	memset(object->u.aes_xts.ivec, 0, 16);
+
+	data_key = GF_CALLOC(1, object->o_dkey_size, gf_crypt_mt_key);
+	if (!data_key)
+		return ENOMEM;
+
+	/*
+	 * retrieve data keying meterial
+	 */
+	ret = get_data_file_key(info, master, object->o_dkey_size, data_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Failed to retrieve data key");
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * parse compound xts key
+	 */
+	subkey_size = object->o_dkey_size >> 4; /* (xts-key-size-in-bytes / 2) */
+	/*
+	 * install key for data encryption
+	 */
+	ret = aes_set_keys_common(data_key,
+				  subkey_size << 3, object->u.aes_xts.dkey);
+	if (ret) {
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * set up key used to encrypt tweaks
+	 */
+	ret = AES_set_encrypt_key(data_key + subkey_size,
+				  object->o_dkey_size / 2,
+				  &object->u.aes_xts.tkey);
+	if (ret < 0)
+		gf_log("crypt", GF_LOG_ERROR, "Set tweak key failed");
+
+	GF_FREE(data_key);
+	return ret;
+}
+
+static int32_t aes_xts_init(void)
+{
+	cassert(AES_BLOCK_SIZE == (1 << AES_BLOCK_BITS));
+	return 0;
+}
+
+static int32_t check_key_aes_xts(uint32_t keysize)
+{
+	switch(keysize) {
+	case 256:
+	case 512:
+		return 0;
+	default:
+		break;
+	}
+	return -1;
+}
+
+static int32_t encrypt_aes_xts(const unsigned char *from,
+			       unsigned char *to, size_t length,
+			       off_t offset, const int enc,
+			       struct object_cipher_info *object)
+{
+	XTS128_CONTEXT ctx;
+	if (enc) {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_ENCRYPT];
+		ctx.block1 = (block128_f)AES_encrypt;
+	}
+	else {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_DECRYPT];
+		ctx.block1 = (block128_f)AES_decrypt;
+	}
+	ctx.key2 = &object->u.aes_xts.tkey;
+	ctx.block2 = (block128_f)AES_encrypt;
+
+	return CRYPTO_xts128_encrypt(&ctx,
+				     object->u.aes_xts.ivec,
+				     from,
+				     to,
+				     length, enc);
+}
+
+/*
+ * Cipher input chunk @from of length @len;
+ * @to: result of cipher transform;
+ * @off: offset in a file (must be cblock-aligned);
+ */
+static void cipher_data(struct object_cipher_info *object,
+			char *from,
+			char *to,
+			off_t off,
+			size_t len,
+			const int enc)
+{
+	crypt_check_input_len(len, object);
+
+#if TRIVIAL_TFM && DEBUG_CRYPT
+	return;
+#endif
+	data_cipher_algs[object->o_alg][object->o_mode].set_iv(off, object);
+	data_cipher_algs[object->o_alg][object->o_mode].encrypt
+		((const unsigned char *)from,
+		 (unsigned char *)to,
+		 len,
+		 off,
+		 enc,
+		 object);
+}
+
+#define MAX_CIPHER_CHUNK (1 << 30)
+
+/*
+ * Do cipher (encryption/decryption) transform of a
+ * continuous region of memory.
+ *
+ * @len: a number of bytes to transform;
+ * @buf: data to transform;
+ * @off: offset in a file, should be block-aligned
+ *       for atomic cipher modes and ksize-aligned
+ *       for other modes).
+ * @dir: direction of transform (encrypt/decrypt).
+ */
+static void cipher_region(struct object_cipher_info *object,
+			  char *from,
+			  char *to,
+			  off_t off,
+			  size_t len,
+			  int dir)
+{
+	while (len > 0) {
+		size_t to_cipher;
+
+		to_cipher = len;
+		if (to_cipher > MAX_CIPHER_CHUNK)
+			to_cipher = MAX_CIPHER_CHUNK;
+
+		/* this will reset IV */
+		cipher_data(object,
+			    from,
+			    to,
+			    off,
+			    to_cipher,
+			    dir);
+		from += to_cipher;
+		to   += to_cipher;
+		off  += to_cipher;
+		len  -= to_cipher;
+	}
+}
+
+/*
+ * Do cipher transform (encryption/decryption) of
+ * plaintext/ciphertext represented by @vec.
+ *
+ * Pre-conditions: @vec represents a continuous piece
+ * of data in a file at offset @off to be ciphered
+ * (encrypted/decrypted).
+ * @count is the number of vec's components. All the
+ * components must be block-aligned, the caller is
+ * responsible for this. @dir is "direction" of
+ * transform (encrypt/decrypt).
+ */
+static void cipher_aligned_iov(struct object_cipher_info *object,
+			       struct iovec *vec,
+			       int count,
+			       off_t off,
+			       int32_t dir)
+{
+	int i;
+	int len = 0;
+
+	for (i = 0; i < count; i++) {
+		cipher_region(object,
+			      vec[i].iov_base,
+			      vec[i].iov_base,
+			      off + len,
+			      vec[i].iov_len,
+			      dir);
+		len += vec[i].iov_len;
+	}
+}
+
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 1);
+}
+
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 0);
+}
+
+#if DEBUG_CRYPT
+static void compound_stream(struct iovec *vec, int count, char *buf, off_t skip)
+{
+	int i;
+	int off = 0;
+	for (i = 0; i < count; i++) {
+		memcpy(buf + off,
+		       vec[i].iov_base + skip,
+		       vec[i].iov_len - skip);
+
+		off += (vec[i].iov_len - skip);
+		skip = 0;
+	}
+}
+
+static void check_iovecs(struct iovec *vec, int cnt,
+			 struct iovec *avec, int acnt, uint32_t off_in_head)
+{
+	char *s1, *s2;
+	uint32_t size, asize;
+
+	size = iovec_get_size(vec, cnt);
+	asize = iovec_get_size(avec, acnt) - off_in_head;
+	if (size != asize) {
+		gf_log("crypt", GF_LOG_DEBUG, "size %d is not eq asize %d",
+		       size, asize);
+		return;
+	}
+	s1 = GF_CALLOC(1, size, gf_crypt_mt_data);
+	if (!s1) {
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	s2 = GF_CALLOC(1, asize, gf_crypt_mt_data);
+	if (!s2) {
+		GF_FREE(s1);
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	compound_stream(vec, cnt, s1, 0);
+	compound_stream(avec, acnt, s2, off_in_head);
+	if (memcmp(s1, s2, size))
+		gf_log("crypt", GF_LOG_DEBUG, "chunks of different data");
+	GF_FREE(s1);
+	GF_FREE(s2);
+}
+
+#else
+#define check_iovecs(vec, count, avec, avecn, off) noop
+#endif /* DEBUG_CRYPT */
+
+static char *data_alloc_block(xlator_t *this, crypt_local_t *local,
+			      int32_t block_size)
+{
+	struct iobuf *iobuf = NULL;
+
+	iobuf = iobuf_get2(this->ctx->iobuf_pool, block_size);
+	if (!iobuf) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Failed to get iobuf");
+		return NULL;
+	}
+	if (!local->iobref_data) {
+		local->iobref_data = iobref_new();
+		if (!local->iobref_data) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "Failed to get iobref");
+			iobuf_unref(iobuf);
+			return NULL;
+		}
+	}
+	iobref_add(local->iobref_data, iobuf);
+	return iobuf->ptr;
+}
+
+/*
+ * Compound @avec, which represent the same data
+ * chunk as @vec, but has aligned components of
+ * specified block size. Alloc blocks, if needed.
+ * In particular, incomplete head and tail blocks
+ * must be allocated.
+ * Put number of allocated blocks to @num_blocks.
+ *
+ * Example:
+ *
+ * input: data chunk represented by 4 components
+ * [AB],[BC],[CD],[DE];
+ * output: 5 logical blocks (0, 1, 2, 3, 4).
+ *
+ *   A     B       C     D               E
+ *   *-----*+------*-+---*----+--------+-*
+ *   |     ||      | |   |    |        | |
+ * *-+-----+*------+-*---+----*--------*-+------*
+ * 0        1        2        3        4
+ *
+ * 0 - incomplete compound (head);
+ * 1, 2 - full compound;
+ * 3 - full non-compound (the case of reuse);
+ * 4 - incomplete non-compound (tail).
+ */
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf)
+{
+	int vecn = 0;      /* number of the current component in vec */
+	int avecn = 0;     /* number of the current component in avec */
+	off_t vec_off = 0; /* offset in the current vec component,
+			    * i.e. the number of bytes have already
+			    * been copied */
+	int32_t block_size = get_atom_size(object);
+	size_t to_process; /* number of vec's bytes to copy and(or) re-use */
+	int32_t off_in_head = conf->off_in_head;
+
+	to_process = iovec_get_size(vec, count);
+
+	while (to_process > 0) {
+		if (off_in_head ||
+		    vec[vecn].iov_len - vec_off < block_size) {
+			/*
+			 * less than block_size:
+			 * the case of incomplete (head or tail),
+			 * or compound block
+			 */
+			size_t copied = 0;
+			/*
+			 * populate the pool with a new block
+			 */
+			blocks[*blocks_allocated] = data_alloc_block(this,
+								    local,
+								    block_size);
+			if (!blocks[*blocks_allocated])
+				return -ENOMEM;
+			memset(blocks[*blocks_allocated], 0, off_in_head);
+			/*
+			 * fill the block with vec components
+			 */
+			do {
+				size_t to_copy;
+
+				to_copy = vec[vecn].iov_len - vec_off;
+				if (to_copy > block_size - off_in_head)
+					to_copy = block_size - off_in_head;
+
+				memcpy(blocks[*blocks_allocated] + off_in_head + copied,
+				       vec[vecn].iov_base + vec_off,
+				       to_copy);
+
+				copied += to_copy;
+				to_process -= to_copy;
+
+				vec_off += to_copy;
+				if (vec_off == vec[vecn].iov_len) {
+					/* finished with this vecn */
+					vec_off = 0;
+					vecn++;
+				}
+			} while (copied < (block_size - off_in_head) && to_process > 0);
+			/*
+			 * update avec
+			 */
+			avec[avecn].iov_len = off_in_head + copied;
+			avec[avecn].iov_base = blocks[*blocks_allocated];
+
+			(*blocks_allocated)++;
+			off_in_head = 0;
+		} else {
+			/*
+			 * the rest of the current vec component
+			 * is not less than block_size, so reuse
+			 * the memory buffer of the component.
+			 */
+			size_t to_reuse;
+			to_reuse = (to_process > block_size ?
+				    block_size :
+				    to_process);
+			avec[avecn].iov_len = to_reuse;
+			avec[avecn].iov_base = vec[vecn].iov_base + vec_off;
+
+			vec_off += to_reuse;
+			if (vec_off == vec[vecn].iov_len) {
+				/* finished with this vecn */
+				vec_off = 0;
+				vecn++;
+			}
+			to_process -= to_reuse;
+		}
+		avecn++;
+	}
+	check_iovecs(vec, count, avec, avecn, conf->off_in_head);
+	return 0;
+}
+
+/*
+ * allocate and setup aligned vector for data submission
+ * Pre-condition: @conf is set.
+ */
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count)
+{
+	int32_t ret = ENOMEM;
+	struct iovec *avec;
+	char **pool;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = DATA_ATOM;
+
+	avec = GF_CALLOC(conf->acount, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ret;
+	pool = GF_CALLOC(conf->acount, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ret;
+	}
+	if (!vec) {
+		/*
+		 * degenerated case: no data
+		 */
+		pool[0] = data_alloc_block(this, local, get_atom_size(object));
+		if (!pool[0])
+			goto free;
+		blocks_in_pool = 1;
+		avec->iov_base = pool[0];
+		avec->iov_len = conf->off_in_tail;
+	}
+	else {
+		ret = align_iov_by_atoms(this, local, object, vec, vec_count,
+					 avec, pool, &blocks_in_pool, conf);
+		if (ret)
+			goto free;
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ret;
+}
+
+/*
+ * allocate and setup aligned vector for hole submission
+ */
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop)
+{
+	uint32_t i, idx;
+	struct iovec *avec;
+	char **pool;
+	uint32_t num_blocks;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = HOLE_ATOM;
+
+	num_blocks = conf->acount -
+		(conf->nr_full_blocks ? conf->nr_full_blocks - 1 : 0);
+
+	switch (fop) {
+	case GF_FOP_WRITE:
+		/*
+		 * hole goes before data
+		 */
+		if (num_blocks == 1 && conf->off_in_tail != 0)
+		/*
+		 * we won't submit a hole which fits into
+		 * a data atom: this part of hole will be
+		 * submitted with data write
+		 */
+			return 0;
+		break;
+	case GF_FOP_FTRUNCATE:
+		/*
+		 * expanding truncate, hole goes after data,
+		 * and will be submited in any case.
+ 		 */
+		break;
+	default:
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad file operation %d", fop);
+		return 0;
+	}
+	avec = GF_CALLOC(num_blocks, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ENOMEM;
+	pool = GF_CALLOC(num_blocks, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ENOMEM;
+	}
+	for (i = 0; i < num_blocks; i++) {
+		pool[i] = data_alloc_block(this, local, get_atom_size(object));
+		if (pool[i] == NULL)
+			goto free;
+		blocks_in_pool++;
+	}
+	if (has_head_block(conf)) {
+		/* set head block */
+		idx = 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base + conf->off_in_head,
+		       0,
+		       get_atom_size(object) - conf->off_in_head);
+	}
+	if (has_tail_block(conf)) {
+		/* set tail block */
+		idx = num_blocks - 1;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base, 0, conf->off_in_tail);
+	}
+	if (has_full_blocks(conf)) {
+		/* set full block */
+		idx = conf->off_in_head ? 1 : 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		/*
+		 * since we re-use the buffer,
+		 * zeroes will be set every time
+		 * before encryption, see submit_full()
+		 */
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ENOMEM;
+}
+
+/* A helper for setting up config of partial atoms (which
+ * participate in read-modify-write sequence).
+ *
+ * Calculate and setup precise amount of "extra-bytes"
+ * that should be uptodated at the end of partial (not
+ * necessarily tail!) block.
+ *
+ * Pre-condition: local->old_file_size is valid!
+ * @conf contains setup, which is enough for correct calculation
+ * of has_tail_block(), ->get_offset().
+ */
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype)
+{
+	uint32_t to_block;
+	crypt_local_t *local = frame->local;
+	uint64_t old_file_size = local->old_file_size;
+	struct rmw_atom *partial = atom_by_types(dtype,
+						 has_tail_block(conf) ?
+						 TAIL_ATOM : HEAD_ATOM);
+
+	if (old_file_size <= partial->offset_at(frame, object))
+		to_block = 0;
+	else {
+		to_block = old_file_size - partial->offset_at(frame, object);
+		if (to_block > get_atom_size(object))
+			to_block = get_atom_size(object);
+	}
+	if (to_block > conf->off_in_tail)
+		conf->gap_in_tail = to_block - conf->off_in_tail;
+	else
+		/*
+		 * nothing to uptodate
+		 */
+		conf->gap_in_tail = 0;
+}
+
+/*
+ * fill struct avec_config with offsets layouts
+ */
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t set_gap)
+{
+	crypt_local_t *local;
+	struct object_cipher_info *object;
+	struct avec_config *conf;
+	uint32_t resid;
+
+	uint32_t atom_size;
+	uint32_t atom_bits;
+
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head = 0;
+	uint32_t off_in_tail = 0;
+	uint32_t nr_full_blocks;
+	int32_t size_full_blocks;
+
+	uint32_t acount; /* number of alifned components to write.
+			  * The same as number of occupied logical
+			  * blocks (atoms)
+			  */
+	local = frame->local;
+	object = &local->info->cinfo;
+	conf = (dtype == DATA_ATOM ?
+		get_data_conf(frame) : get_hole_conf(frame));
+
+	orig_offset = offset;
+	orig_size = count;
+
+	atom_size = get_atom_size(object);
+	atom_bits = get_atom_bits(object);
+
+	/*
+	 * Round-down the start,
+	 * round-up the end.
+	 */
+	resid = offset & (uint64_t)(atom_size - 1);
+
+	if (resid)
+		off_in_head = resid;
+	aligned_offset = offset - off_in_head;
+	expanded_size = orig_size + off_in_head;
+
+	/* calculate tail,
+	   expand size forward  */
+	resid = (offset + orig_size) & (uint64_t)(atom_size - 1);
+
+	if (resid) {
+		off_in_tail = resid;
+		expanded_size += (atom_size - off_in_tail);
+	}
+	/*
+	 * calculate number of occupied blocks
+	 */
+	acount = expanded_size >> atom_bits;
+	/*
+	 * calculate number of full blocks
+	 */
+	size_full_blocks = expanded_size;
+	if (off_in_head)
+		size_full_blocks -= atom_size;
+	if (off_in_tail && size_full_blocks > 0)
+		size_full_blocks -= atom_size;
+	nr_full_blocks = size_full_blocks >> atom_bits;
+
+	conf->atom_size = atom_size;
+	conf->orig_size = orig_size;
+	conf->orig_offset = orig_offset;
+	conf->expanded_size = expanded_size;
+	conf->aligned_offset = aligned_offset;
+
+	conf->off_in_head = off_in_head;
+	conf->off_in_tail = off_in_tail;
+	conf->nr_full_blocks = nr_full_blocks;
+	conf->acount = acount;
+	/*
+	 * Finally, calculate precise amount of
+	 * "extra-bytes" that should be uptodated
+	 * at the end.
+	 * Only if RMW is expected.
+	 */
+	if (off_in_tail && set_gap)
+		set_gap_at_end(frame, object, conf, dtype);
+}
+
+struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE] = {
+	[AES_CIPHER_ALG][XTS_CIPHER_MODE] =
+	{ .atomic = _gf_true,
+	  .should_pad = _gf_true,
+	  .blkbits = AES_BLOCK_BITS,
+	  .init = aes_xts_init,
+	  .set_private = set_private_aes_xts,
+	  .check_key = check_key_aes_xts,
+	  .set_iv = set_iv_aes_xts,
+	  .encrypt = encrypt_aes_xts
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/keys.c b/xlators/encryption/crypt/src/keys.c
new file mode 100644
index 000000000..4a1d3bb5a
--- /dev/null
+++ b/xlators/encryption/crypt/src/keys.c
@@ -0,0 +1,302 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/* Key hierarchy
+                  
+                   +----------------+
+                   | MASTER_VOL_KEY |
+                   +-------+--------+
+                           |
+                           |
+          +----------------+----------------+
+          |                |                |
+          |                |                |
+  +-------+------+ +-------+-------+ +------+--------+
+  | NMTD_VOL_KEY | | EMTD_FILE_KEY | | DATA_FILE_KEY |
+  +-------+------+ +---------------+ +---------------+
+          |
+          |
+  +-------+-------+
+  | NMTD_LINK_KEY |
+  +---------------+
+
+ */
+
+#if DEBUG_CRYPT
+static void check_prf_iters(uint32_t num_iters)
+{
+	if (num_iters == 0)
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad number of prf iterations : %d", num_iters);
+}
+#else
+#define check_prf_iters(num_iters) noop
+#endif /* DEBUG_CRYPT */
+
+unsigned char crypt_fake_oid[16] =
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+/*
+ * derive key in the counter mode using
+ * sha256-based HMAC as PRF, see
+ * NIST Special Publication 800-108, 5.1)
+ */
+
+#define PRF_OUTPUT_SIZE  SHA256_DIGEST_LENGTH
+
+static int32_t kderive_init(struct kderive_context *ctx,
+			    const unsigned char *pkey, /* parent key */
+			    uint32_t pkey_size,  /* parent key size */
+			    const unsigned char *idctx, /* id-context */
+			    uint32_t idctx_size,
+			    crypt_key_type type  /* type of child key */)
+{
+	unsigned char *pos;
+	uint32_t llen = strlen(crypt_keys[type].label);	
+	/*
+	 * Compoud the fixed input data for KDF:
+	 * [i]_2 || Label || 0x00 || Id-Context || [L]_2),
+	 * NIST SP 800-108, 5.1
+	 */
+	ctx->fid_len =
+		sizeof(uint32_t) +
+		llen +
+		1 +
+		idctx_size +
+		sizeof(uint32_t);
+
+	ctx->fid = GF_CALLOC(ctx->fid_len, 1, gf_crypt_mt_key);
+	if (!ctx->fid)
+		return ENOMEM;
+	ctx->out_len = round_up(crypt_keys[type].len >> 3,
+				PRF_OUTPUT_SIZE);
+	ctx->out = GF_CALLOC(ctx->out_len, 1, gf_crypt_mt_key);
+	if (!ctx->out) {
+		GF_FREE(ctx->fid);
+		return ENOMEM;
+	}
+	ctx->pkey = pkey;
+	ctx->pkey_len = pkey_size;
+	ctx->ckey_len = crypt_keys[type].len;
+
+	pos = ctx->fid;
+
+	/* counter will be set up in kderive_rfn() */
+	pos += sizeof(uint32_t);
+
+	memcpy(pos, crypt_keys[type].label, llen);
+	pos += llen;
+
+	/* set up zero octet */
+	*pos = 0;
+	pos += 1;
+
+	memcpy(pos, idctx, idctx_size);
+	pos += idctx_size;
+
+	*((uint32_t *)pos) = htobe32(ctx->ckey_len);
+
+	return 0;
+}
+
+static void kderive_update(struct kderive_context *ctx)
+{	
+	uint32_t i;
+	HMAC_CTX hctx;
+	unsigned char *pos = ctx->out;
+	uint32_t *p_iter = (uint32_t *)ctx->fid;
+	uint32_t num_iters = ctx->out_len / PRF_OUTPUT_SIZE;
+
+	check_prf_iters(num_iters);
+
+	HMAC_CTX_init(&hctx);
+	for (i = 0; i < num_iters; i++) {
+		/*
+		 * update the iteration number in the fid
+		 */
+		*p_iter = htobe32(i);
+		HMAC_Init_ex(&hctx,
+			     ctx->pkey, ctx->pkey_len >> 3,
+			     EVP_sha256(),
+			     NULL);
+		HMAC_Update(&hctx, ctx->fid, ctx->fid_len);
+		HMAC_Final(&hctx, pos, NULL);
+
+		pos += PRF_OUTPUT_SIZE;
+	}
+	HMAC_CTX_cleanup(&hctx);
+}
+
+static void kderive_final(struct kderive_context *ctx, unsigned char *child)
+{
+	memcpy(child, ctx->out, ctx->ckey_len >> 3);
+	GF_FREE(ctx->fid);
+	GF_FREE(ctx->out);
+	memset(ctx, 0, sizeof(*ctx));
+}
+
+/*
+ * derive per-volume key for object ids aithentication
+ */
+int32_t get_nmtd_vol_key(struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   crypt_fake_oid, sizeof(uuid_t), NMTD_VOL_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, master->m_nmtd_key);
+	return 0;
+}
+
+/*
+ * derive per-link key for aithentication of non-encrypted
+ * meta-data (nmtd)
+ */
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_nmtd_key,
+			   nmtd_vol_key_size(),
+			   (const unsigned char *)loc->path,
+			   strlen(loc->path), NMTD_LINK_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+/*
+ * derive per-file key for encryption and authentication
+ * of encrypted part of metadata (emtd)
+ */
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), EMTD_FILE_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+static int32_t data_key_type_by_size(uint32_t keysize, crypt_key_type *type)
+{
+	int32_t ret = 0;
+	switch (keysize) {
+	case 256:
+		*type = DATA_FILE_KEY_256;
+		break;
+	case 512:
+		*type = DATA_FILE_KEY_512;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Unsupported data key size %d",
+		       keysize);
+		ret = ENOTSUP;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * derive per-file key for data encryption
+ */
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+	crypt_key_type type;
+
+	ret = data_key_type_by_size(keysize, &type);
+	if (ret)
+		return ret;
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), type);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, key);
+	return 0;
+}
+
+/*
+ * NOTE: Don't change existing keys: it will break compatibility;
+ */
+struct crypt_key crypt_keys[LAST_KEY_TYPE] = {
+	[MASTER_VOL_KEY] =
+	{ .len = MASTER_VOL_KEY_SIZE << 3,
+	  .label = "volume-master",
+	},
+	[NMTD_VOL_KEY] =
+	{ .len = NMTD_VOL_KEY_SIZE << 3,
+	  .label = "volume-nmtd-key-generation"
+	},
+	[NMTD_LINK_KEY] =
+	{ .len = 128,
+	  .label = "link-nmtd-authentication"
+	},
+	[EMTD_FILE_KEY] =
+	{ .len = 128,
+	  .label = "file-emtd-encryption-and-auth"
+	},
+	[DATA_FILE_KEY_256] =
+	{ .len = 256,
+	  .label = "file-data-encryption-256"
+	},
+	[DATA_FILE_KEY_512] =
+	{ .len = 512,
+	  .label = "file-data-encryption-512"
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.c b/xlators/encryption/crypt/src/metadata.c
new file mode 100644
index 000000000..36b14c055
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.c
@@ -0,0 +1,605 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+#include "metadata.h"
+
+int32_t alloc_format(crypt_local_t *local, size_t size)
+{
+	if (size > 0) {
+		local->format = GF_CALLOC(1, size, gf_crypt_mt_mtd);
+		if (!local->format)
+			return ENOMEM;
+	}
+	local->format_size = size;
+	return 0;
+}
+
+int32_t alloc_format_create(crypt_local_t *local)
+{
+	return alloc_format(local, new_format_size());
+}
+
+void free_format(crypt_local_t *local)
+{
+	GF_FREE(local->format);
+}
+
+/*
+ * Check compatibility with extracted metadata
+ */
+static int32_t check_file_metadata(struct crypt_inode_info *info)
+{
+	struct object_cipher_info *object = &info->cinfo;
+
+	if (info->nr_minor != CRYPT_XLATOR_ID) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported minor subversion %d", info->nr_minor);
+		return EINVAL;
+	}
+	if (object->o_alg > LAST_CIPHER_ALG) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher algorithm %d",
+		       object->o_alg);
+		return EINVAL;
+	}
+	if (object->o_mode > LAST_CIPHER_MODE) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher mode %d",
+		       object->o_mode);
+		return EINVAL;
+	}
+	if (object->o_block_bits < CRYPT_MIN_BLOCK_BITS ||
+	    object->o_block_bits > CRYPT_MAX_BLOCK_BITS) {
+		gf_log("crypt", GF_LOG_WARNING, "unsupported block bits %d",
+		       object->o_block_bits);
+		return EINVAL;
+	}
+	/* TBD: check data key size */
+	return 0;
+}
+
+static size_t format_size_v1(mtd_op_t op, size_t old_size)
+{
+	
+	switch (op) {
+	case MTD_CREATE:
+		return sizeof(struct mtd_format_v1);
+	case MTD_OVERWRITE:
+		return old_size;
+	case MTD_APPEND:
+		return old_size + NMTD_8_MAC_SIZE;
+	case MTD_CUT:
+		if (old_size > sizeof(struct mtd_format_v1))
+			return old_size - NMTD_8_MAC_SIZE;
+		else
+			return 0;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad mtd operation");
+		return 0;
+	}
+}
+
+/*
+ * Calculate size of the updated format string.
+ * Returned zero means that we don't need to update the format string.
+ */
+size_t format_size(mtd_op_t op, size_t old_size)
+{
+	size_t versioned;
+
+	versioned = mtd_loaders[current_mtd_loader()].format_size(op,
+				 old_size - sizeof(struct crypt_format));
+	if (versioned != 0)
+		return versioned + sizeof(struct crypt_format);
+	return 0;
+}
+
+/*
+ * size of the format string of newly created file (nr_links = 1)
+ */
+size_t new_format_size(void)
+{
+	return format_size(MTD_CREATE, 0);
+}
+
+/*
+ * Calculate per-link MAC by pathname
+ */
+static int32_t calc_link_mac_v1(struct mtd_format_v1 *fmt,
+				loc_t *loc,
+				unsigned char *result,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;	
+	unsigned char nmtd_link_key[16];
+	CMAC_CTX *cctx;
+	size_t len;
+
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not get nmtd link key");
+		return -1;
+	}
+	cctx = CMAC_CTX_new();
+	if (!cctx) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_CTX_new failed");
+		return -1; 
+	}
+	ret = CMAC_Init(cctx, nmtd_link_key, sizeof(nmtd_link_key),
+			EVP_aes_128_cbc(), 0);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Init failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Update(cctx, get_NMTD_V1(info), SIZE_OF_NMTD_V1);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Update failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Final(cctx, result, &len);
+	CMAC_CTX_free(cctx);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Final failed");
+		return -1;
+	}
+	return 0;
+}
+
+/*
+ * Create per-link MAC of index @idx by pathname
+ */
+static int32_t create_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	memcpy(mac, cmac, SIZE_OF_NMTD_V1_MAC);
+	return 0;
+}
+
+static int32_t create_format_v1(unsigned char *wire,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	unsigned char nmtd_link_key[16];
+	uint32_t ad;
+	GCM128_CONTEXT *gctx;
+
+	fmt = (struct mtd_format_v1 *)wire;
+
+	fmt->minor_id = info->nr_minor;
+	fmt->alg_id = AES_CIPHER_ALG;
+	fmt->dkey_factor = master->m_dkey_size >> KEY_FACTOR_BITS;
+	fmt->block_bits = master->m_block_bits;
+	fmt->mode_id = master->m_mode;
+	/*
+	 * retrieve keys for the parts of metadata
+	 */
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret)
+		return ret;
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret)
+		return ret;
+	
+	AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+
+	/* TBD: Check return values */
+
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	ret = CRYPTO_gcm128_encrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_encrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * set MAC of encrypted part of metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC);
+	CRYPTO_gcm128_release(gctx);
+	/*
+	 * set the first MAC of non-encrypted part of metadata 
+	 */
+	return create_link_mac_v1(fmt, 0, loc, info, master);
+}
+
+/*
+ * Called by fops:
+ * ->create();
+ * ->link();
+ *
+ * Pack common and version-specific parts of file's metadata
+ * Pre-conditions: @info contains valid object-id.
+ */
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master)
+{
+	struct crypt_format *fmt = (struct crypt_format *)wire;
+
+	fmt->loader_id = current_mtd_loader();
+
+	wire += sizeof(struct crypt_format);
+	return mtd_loaders[current_mtd_loader()].create_format(wire, loc,
+							       info, master);
+}
+
+/*
+ * Append or overwrite per-link mac of @mac_idx index
+ * in accordance with the new pathname
+ */
+int32_t appov_link_mac_v1(unsigned char *new,
+			  unsigned char *old,
+			  uint32_t old_size,
+			  int32_t mac_idx,
+			  loc_t *loc,
+			  struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  crypt_local_t *local)
+{
+	memcpy(new, old, old_size);
+	return create_link_mac_v1((struct mtd_format_v1 *)new, mac_idx,
+				  loc, info, master);
+}
+
+/*
+ * Cut per-link mac of @mac_idx index
+ */
+static int32_t cut_link_mac_v1(unsigned char *new,
+			       unsigned char *old,			       
+			       uint32_t old_size,
+			       int32_t mac_idx,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local)
+{
+	memcpy(new,
+	       old,
+	       sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1));
+
+	memcpy(new + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1),
+	       old + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx,
+	       old_size - (sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx));
+	return 0;
+}
+
+int32_t update_format_v1(unsigned char *new,
+			 unsigned char *old,
+				size_t old_len,
+				int32_t mac_idx, /* of old name */
+				mtd_op_t op,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master,
+				crypt_local_t *local)
+{
+	switch (op) {
+	case MTD_APPEND:
+		mac_idx = 1 + (old_len - sizeof(struct mtd_format_v1))/8;
+	case MTD_OVERWRITE:
+		return appov_link_mac_v1(new, old, old_len, mac_idx,
+					 loc, info, master, local);
+	case MTD_CUT:
+		return cut_link_mac_v1(new, old, old_len, mac_idx,
+				       loc, info, master, local);
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad  mtd operation %d", op);
+		return -1;
+	}
+}
+
+/*
+ * Called by fops:
+ *
+ * ->link()
+ * ->unlink()
+ * ->rename()
+ *
+ */
+int32_t update_format(unsigned char *new,
+		      unsigned char *old,
+		      size_t old_len,
+		      int32_t mac_idx,
+		      mtd_op_t op,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local)
+{
+	if (!new)
+		return 0;
+	memcpy(new, old, sizeof(struct crypt_format));
+
+	old += sizeof(struct crypt_format);
+	new += sizeof(struct crypt_format);
+	old_len -= sizeof(struct crypt_format);
+
+	return mtd_loaders[current_mtd_loader()].update_format(new, old,
+							       old_len,
+							       mac_idx, op,
+							       loc, info,
+							       master, local);
+}
+
+/*
+ * Perform preliminary checks of found metadata
+ * Return < 0 on errors;
+ * Return number of object-id MACs (>= 1) on success
+ */
+int32_t check_format_v1(uint32_t len, unsigned char *wire)
+{
+	uint32_t nr_links;
+
+	if (len < sizeof(struct mtd_format_v1)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata size %d", len);
+		goto error;
+	}
+	len -= sizeof(struct mtd_format_v1);
+	if (len % sizeof(nmtd_8_mac_t)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata format");
+		goto error;
+	}
+	nr_links = 1 + len / sizeof(nmtd_8_mac_t);
+	if (nr_links > _POSIX_LINK_MAX)
+		goto error;
+	return nr_links;
+ error:
+	return EIO;
+}
+
+/*
+ * Verify per-link MAC specified by index @idx
+ *
+ * return:
+ * -1 on errors;
+ *  0 on failed verification;
+ *  1 on sucessful verification
+ */
+static int32_t verify_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx /* index of the mac to verify */,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	if (memcmp(cmac, mac, SIZE_OF_NMTD_V1_MAC))
+		return 0;
+	return 1;
+}
+
+/*
+ * Lookup per-link MAC by pathname.
+ *
+ * return index of the MAC, if it was found;
+ * return < 0 on errors, or if the MAC wasn't found
+ */
+static int32_t lookup_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t nr_macs,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	uint32_t idx;
+
+	for (idx = 0; idx < nr_macs; idx++) {
+		ret = verify_link_mac_v1(fmt, idx, loc, info, master);
+		if (ret < 0)
+			return ret;
+		if (ret > 0)
+			return idx;
+	}
+	return -ENOENT;
+}
+
+/*
+ * Extract version-specific part of metadata
+ */
+static int32_t open_format_v1(unsigned char *wire,
+			      int32_t len,
+			      loc_t *loc,
+			      struct crypt_inode_info *info,
+			      struct master_cipher_info *master,
+			      crypt_local_t *local,
+			      gf_boolean_t load_info)
+{
+	int32_t ret;
+	int32_t num_nmtd_macs;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	GCM128_CONTEXT *gctx;
+	uint32_t ad;
+	emtd_8_mac_t gmac;
+	struct object_cipher_info *object;
+
+	num_nmtd_macs = check_format_v1(len, wire);
+	if (num_nmtd_macs <= 0)
+		return EIO;
+	fmt = (struct mtd_format_v1 *)wire;
+
+	ret = lookup_link_mac_v1(fmt, num_nmtd_macs, loc, info, master);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "NMTD verification failed");
+		return EINVAL;
+	}
+	local->mac_idx = ret;
+	if (load_info == _gf_false)
+		/* the case of partial open */
+		return 0;
+
+	object = &info->cinfo;
+
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not retrieve metadata key");
+		return ret;
+	}
+	/*
+	 * decrypt encrypted meta-data
+	 */
+	ret = AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not set encrypt key");
+		return ret;
+	}
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+	if (!gctx) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not alloc gcm context");
+		return ENOMEM;
+	}
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;	
+	}
+	ret = CRYPTO_gcm128_decrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_decrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * verify metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, gmac, sizeof(gmac));
+	CRYPTO_gcm128_release(gctx);
+	if (memcmp(gmac, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC)) {
+		gf_log("crypt", GF_LOG_ERROR, "EMTD verification failed");
+		return EINVAL;
+	}
+	/*
+	 * load verified metadata to the private part of inode
+	 */
+	info->nr_minor = fmt->minor_id;
+
+	object->o_alg = fmt->alg_id;
+	object->o_dkey_size = fmt->dkey_factor << KEY_FACTOR_BITS;
+	object->o_block_bits = fmt->block_bits;
+	object->o_mode = fmt->mode_id;
+
+	return check_file_metadata(info);
+}
+
+/*
+ * perform metadata authentication against @loc->path;
+ * extract crypt-specific attribtes and populate @info
+ * with them (optional)
+ */
+int32_t open_format(unsigned char *str,
+		    int32_t len,
+		    loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master,
+		    crypt_local_t *local,
+		    gf_boolean_t load_info)
+{
+	struct crypt_format *fmt;
+	if (len < sizeof(*fmt)) {
+		gf_log("crypt", GF_LOG_ERROR, "Bad core format");
+		return EIO;
+	}
+	fmt = (struct crypt_format *)str;
+
+	if (fmt->loader_id >= LAST_MTD_LOADER) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Unsupported loader id %d", fmt->loader_id);
+		return EINVAL;
+	}
+	str += sizeof(*fmt);
+	len -= sizeof(*fmt);
+
+	return mtd_loaders[fmt->loader_id].open_format(str,
+						       len,
+						       loc,
+						       info,
+						       master,
+						       local,
+						       load_info);
+}
+
+struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER] = {
+	[MTD_LOADER_V1] = 
+	{.format_size = format_size_v1,
+	 .create_format = create_format_v1,
+	 .open_format = open_format_v1,
+	 .update_format = update_format_v1
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.h b/xlators/encryption/crypt/src/metadata.h
new file mode 100644
index 000000000..a92f149ef
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.h
@@ -0,0 +1,74 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __METADATA_H__
+#define __METADATA_H__
+
+#define NMTD_8_MAC_SIZE   (8)
+#define EMTD_8_MAC_SIZE   (8)
+
+typedef uint8_t nmtd_8_mac_t[NMTD_8_MAC_SIZE];
+typedef uint8_t emtd_8_mac_t[EMTD_8_MAC_SIZE] ;
+
+/*
+ * Version "v1" of file's metadata.
+ * Metadata of this version has 4 components:
+ * 
+ * 1) EMTD (Encrypted part of MeTaData);
+ * 2) NMTD (Non-encrypted part of MeTaData);
+ * 3) EMTD_MAC; (EMTD Message Authentication Code);
+ * 4) Array of per-link NMTD MACs (for every (hard)link it includes
+ *    exactly one MAC)
+ */
+struct mtd_format_v1 {
+	/* EMTD, encrypted part of meta-data */
+	uint8_t            alg_id;  /* cipher algorithm id (only AES for now) */
+        uint8_t            mode_id; /* cipher mode id; (only XTS for now) */
+	uint8_t            block_bits; /* encoded block size */
+	uint8_t            minor_id; /* client translator id */
+	uint8_t            dkey_factor; /* encoded size of the data key */
+	/* MACs */
+	emtd_8_mac_t       gmac; /* MAC of the encrypted meta-data, 8 bytes */
+	nmtd_8_mac_t       omac; /* per-link MACs of the non-encrypted
+				  * meta-data: at least one such MAC is always
+				  * present */
+} __attribute__((packed));
+
+/*
+ * NMTD, the non-encrypted part of metadata of version "v1"
+ * is file's gfid, which is generated on trusted machines.
+ */
+#define SIZE_OF_NMTD_V1 (sizeof(uuid_t))
+#define SIZE_OF_EMTD_V1 (offsetof(struct mtd_format_v1, gmac) -		\
+			 offsetof(struct mtd_format_v1, alg_id))
+#define SIZE_OF_NMTD_V1_MAC (NMTD_8_MAC_SIZE)
+#define SIZE_OF_EMTD_V1_MAC (EMTD_8_MAC_SIZE)
+
+static inline unsigned char *get_EMTD_V1(struct mtd_format_v1 *format)
+{
+        return &format->alg_id;
+}
+
+static inline unsigned char *get_NMTD_V1(struct crypt_inode_info *info)
+{
+        return info->oid;
+}
+
+static inline unsigned char *get_EMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->gmac;
+}
+
+static inline unsigned char *get_NMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->omac;
+}
+
+#endif /* __METADATA_H__ */
diff --git a/xlators/encryption/rot-13/src/Makefile.am b/xlators/encryption/rot-13/src/Makefile.am
index ba5e623d8..94e8d18e7 100644
--- a/xlators/encryption/rot-13/src/Makefile.am
+++ b/xlators/encryption/rot-13/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = rot-13.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
 
-rot_13_la_LDFLAGS = -module -avoidversion
+rot_13_la_LDFLAGS = -module -avoid-version
 
 rot_13_la_SOURCES = rot-13.c
 rot_13_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = rot-13.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/encryption/rot-13/src/rot-13.c b/xlators/encryption/rot-13/src/rot-13.c
index 7cae46134..b9ac29a72 100644
--- a/xlators/encryption/rot-13/src/rot-13.c
+++ b/xlators/encryption/rot-13/src/rot-13.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <ctype.h>
 #include <sys/uio.h>
 
@@ -32,13 +22,13 @@
 #include "rot-13.h"
 
 /*
- * This is a rot13 ``encryption'' xlator. It rot13's data when 
- * writing to disk and rot13's it back when reading it. 
+ * This is a rot13 ``encryption'' xlator. It rot13's data when
+ * writing to disk and rot13's it back when reading it.
  * This xlator is meant as an example, NOT FOR PRODUCTION
  * USE ;) (hence no error-checking)
  */
 
-void 
+void
 rot13 (char *buf, int len)
 {
 	int i;
@@ -67,14 +57,16 @@ rot13_readv_cbk (call_frame_t *frame,
                  int32_t op_errno,
                  struct iovec *vector,
                  int32_t count,
-		 struct stat *stbuf)
+		 struct iatt *stbuf,
+                 struct iobref *iobref, dict_t *xdata)
 {
 	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-  
+
 	if (priv->decrypt_read)
 		rot13_iovec (vector, count);
 
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf);
+	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
+                             stbuf, iobref, xdata);
 	return 0;
 }
 
@@ -83,13 +75,13 @@ rot13_readv (call_frame_t *frame,
              xlator_t *this,
              fd_t *fd,
              size_t size,
-             off_t offset)
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
 	STACK_WIND (frame,
 		    rot13_readv_cbk,
 		    FIRST_CHILD (this),
 		    FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset);
+		    fd, size, offset, flags, xdata);
 	return 0;
 }
 
@@ -99,9 +91,11 @@ rot13_writev_cbk (call_frame_t *frame,
                   xlator_t *this,
                   int32_t op_ret,
                   int32_t op_errno,
-		  struct stat *stbuf)
+                  struct iatt *prebuf,
+		  struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, stbuf);
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 	return 0;
 }
 
@@ -110,18 +104,20 @@ rot13_writev (call_frame_t *frame,
               xlator_t *this,
               fd_t *fd,
               struct iovec *vector,
-              int32_t count, 
-              off_t offset)
+              int32_t count,
+              off_t offset, uint32_t flags,
+              struct iobref *iobref, dict_t *xdata)
 {
 	rot_13_private_t *priv = (rot_13_private_t *)this->private;
 	if (priv->encrypt_write)
 		rot13_iovec (vector, count);
 
-	STACK_WIND (frame, 
+	STACK_WIND (frame,
 		    rot13_writev_cbk,
 		    FIRST_CHILD (this),
 		    FIRST_CHILD (this)->fops->writev,
-		    fd, vector, count, offset);
+		    fd, vector, count, offset, flags,
+                    iobref, xdata);
 	return 0;
 }
 
@@ -132,7 +128,7 @@ init (xlator_t *this)
 	rot_13_private_t *priv = NULL;
 
 	if (!this->children || this->children->next) {
-		gf_log ("rot13", GF_LOG_ERROR, 
+		gf_log ("rot13", GF_LOG_ERROR,
 			"FATAL: rot13 should have exactly one child");
 		return -1;
 	}
@@ -141,9 +137,11 @@ init (xlator_t *this)
 		gf_log (this->name, GF_LOG_WARNING,
 			"dangling volume. check volfile ");
 	}
-  
-	priv = CALLOC (sizeof (rot_13_private_t), 1);
-	ERR_ABORT (priv);
+
+	priv = GF_CALLOC (sizeof (rot_13_private_t), 1, 0);
+        if (!priv)
+                return -1;
+
 	priv->decrypt_read = 1;
 	priv->encrypt_write = 1;
 
@@ -170,13 +168,16 @@ init (xlator_t *this)
 	return 0;
 }
 
-void 
+void
 fini (xlator_t *this)
 {
 	rot_13_private_t *priv = this->private;
-	
-	FREE (priv);
-	
+
+        if (!priv)
+                return;
+        this->private = NULL;
+	GF_FREE (priv);
+
 	return;
 }
 
@@ -185,16 +186,14 @@ struct xlator_fops fops = {
 	.writev       = rot13_writev
 };
 
-struct xlator_mops mops = {
-};
-
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-	{ .key  = {"encrypt-write"}, 
+	{ .key  = {"encrypt-write"},
 	  .type = GF_OPTION_TYPE_BOOL
 	},
-	{ .key  = {"decrypt-read"}, 
-	  .type = GF_OPTION_TYPE_BOOL 
+	{ .key  = {"decrypt-read"},
+	  .type = GF_OPTION_TYPE_BOOL
 	},
 	{ .key  = {NULL} },
 };
diff --git a/xlators/encryption/rot-13/src/rot-13.h b/xlators/encryption/rot-13/src/rot-13.h
index 43e60c326..3e9fc19c7 100644
--- a/xlators/encryption/rot-13/src/rot-13.h
+++ b/xlators/encryption/rot-13/src/rot-13.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __ROT_13_H__
 #define __ROT_13_H__
 
diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index 9ac9b6f19..d2f5ef192 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = locks trash path-convertor filter quota
+SUBDIRS = locks quota read-only mac-compat quiesce marker index \
+	  protect compress changelog gfid-access $(GLUPY_SUBDIR) qemu-block # trash path-converter # filter
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/changelog/Makefile.am b/xlators/features/changelog/Makefile.am
new file mode 100644
index 000000000..153bb6850
--- /dev/null
+++ b/xlators/features/changelog/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src lib
+
+CLEANFILES =
diff --git a/xlators/features/changelog/lib/Makefile.am b/xlators/features/changelog/lib/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/changelog/lib/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/changelog/lib/examples/c/get-changes.c b/xlators/features/changelog/lib/examples/c/get-changes.c
new file mode 100644
index 000000000..14562585a
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/c/get-changes.c
@@ -0,0 +1,87 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+/**
+ * get set of new changes every 10 seconds (just print the file names)
+ *
+ * Compile it using:
+ *  gcc -o getchanges `pkg-config --cflags libgfchangelog` get-changes.c \
+ *  `pkg-config --libs libgfchangelog`
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <limits.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <errno.h>
+
+#include "changelog.h"
+
+#define handle_error(fn)                                \
+        printf ("%s (reason: %s)\n", fn, strerror (errno))
+
+int
+main (int argc, char ** argv)
+{
+        int     i           = 0;
+        int     ret         = 0;
+        ssize_t nr_changes  = 0;
+        ssize_t changes     = 0;
+        char fbuf[PATH_MAX] = {0,};
+
+        /* get changes for brick "/home/vshankar/export/yow/yow-1" */
+        ret = gf_changelog_register ("/home/vshankar/export/yow/yow-1",
+                                     "/tmp/scratch", "/tmp/change.log", 9, 5);
+        if (ret) {
+                handle_error ("register failed");
+                goto out;
+        }
+
+        while (1) {
+                i = 0;
+                nr_changes = gf_changelog_scan ();
+                if (nr_changes < 0) {
+                        handle_error ("scan(): ");
+                        break;
+                }
+
+                if (nr_changes == 0)
+                        goto next;
+
+                printf ("Got %ld changelog files\n", nr_changes);
+
+                while ( (changes =
+                         gf_changelog_next_change (fbuf, PATH_MAX)) > 0) {
+                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
+
+                        /* process changelog */
+                        /* ... */
+                        /* ... */
+                        /* ... */
+                        /* done processing */
+
+                        ret = gf_changelog_done (fbuf);
+                        if (ret)
+                                handle_error ("gf_changelog_done");
+                }
+
+                if (changes == -1)
+                        handle_error ("gf_changelog_next_change");
+
+        next:
+                sleep (10);
+        }
+
+ out:
+        return ret;
+}
diff --git a/xlators/features/changelog/lib/examples/python/changes.py b/xlators/features/changelog/lib/examples/python/changes.py
new file mode 100644
index 000000000..d21db8eab
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/python/changes.py
@@ -0,0 +1,32 @@
+#!/usr/bin/python
+
+import os
+import sys
+import time
+import libgfchangelog
+
+cl = libgfchangelog.Changes()
+
+def get_changes(brick, scratch_dir, log_file, log_level, interval):
+    change_list = []
+    try:
+        cl.cl_register(brick, scratch_dir, log_file, log_level)
+        while True:
+            cl.cl_scan()
+            change_list = cl.cl_getchanges()
+            if change_list:
+                print change_list
+            for change in change_list:
+                print('done with %s' % (change))
+                cl.cl_done(change)
+            time.sleep(interval)
+    except OSError:
+        ex = sys.exc_info()[1]
+        print ex
+
+if __name__ == '__main__':
+    if len(sys.argv) != 5:
+        print("usage: %s <brick> <scratch-dir> <log-file> <fetch-interval>"
+              % (sys.argv[0]))
+        sys.exit(1)
+    get_changes(sys.argv[1], sys.argv[2], sys.argv[3], 9, int(sys.argv[4]))
diff --git a/xlators/features/changelog/lib/examples/python/libgfchangelog.py b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
new file mode 100644
index 000000000..68ec3baf1
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
@@ -0,0 +1,64 @@
+import os
+from ctypes import *
+from ctypes.util import find_library
+
+class Changes(object):
+    libgfc = CDLL(find_library("gfchangelog"), use_errno=True)
+
+    @classmethod
+    def geterrno(cls):
+        return get_errno()
+
+    @classmethod
+    def raise_oserr(cls):
+        errn = cls.geterrno()
+        raise OSError(errn, os.strerror(errn))
+
+    @classmethod
+    def _get_api(cls, call):
+        return getattr(cls.libgfc, call)
+
+    @classmethod
+    def cl_register(cls, brick, path, log_file, log_level, retries = 0):
+        ret = cls._get_api('gf_changelog_register')(brick, path,
+                                                    log_file, log_level, retries)
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_scan(cls):
+        ret = cls._get_api('gf_changelog_scan')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_startfresh(cls):
+        ret = cls._get_api('gf_changelog_start_fresh')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_getchanges(cls):
+        """ remove hardcoding for path name length """
+        def clsort(f):
+            return f.split('.')[-1]
+        changes = []
+        buf = create_string_buffer('\0', 4096)
+        call = cls._get_api('gf_changelog_next_change')
+
+        while True:
+            ret = call(buf, 4096)
+            if ret in (0, -1):
+                break;
+            changes.append(buf.raw[:ret-1])
+        if ret == -1:
+            cls.raise_oserr()
+        # cleanup tracker
+        cls.cl_startfresh()
+        return sorted(changes, key=clsort)
+
+    @classmethod
+    def cl_done(cls, clfile):
+        ret = cls._get_api('gf_changelog_done')(clfile)
+        if ret == -1:
+            cls.raise_oserr()
diff --git a/xlators/features/changelog/lib/src/Makefile.am b/xlators/features/changelog/lib/src/Makefile.am
new file mode 100644
index 000000000..fbaaea628
--- /dev/null
+++ b/xlators/features/changelog/lib/src/Makefile.am
@@ -0,0 +1,37 @@
+libgfchangelog_la_CFLAGS = -Wall $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
+			-DDATADIR=\"$(localstatedir)\"
+
+libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -fpic \
+				-I../../../src/ -I$(top_srcdir)/libglusterfs/src \
+				-I$(top_srcdir)/xlators/features/changelog/src \
+				-DDATADIR=\"$(localstatedir)\"
+
+libgfchangelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+			$(GF_GLUSTERFS_LIBS)
+
+libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS)
+
+libgfchangelogdir = $(includedir)/glusterfs/gfchangelog
+lib_LTLIBRARIES = libgfchangelog.la
+
+CONTRIB_BUILDDIR = $(top_builddir)/contrib
+
+libgfchangelog_la_SOURCES = gf-changelog.c gf-changelog-process.c \
+		gf-changelog-helpers.c $(CONTRIBDIR)/uuid/clear.c \
+		$(CONTRIBDIR)/uuid/copy.c $(CONTRIBDIR)/uuid/gen_uuid.c \
+		$(CONTRIBDIR)/uuid/pack.c $(CONTRIBDIR)/uuid/parse.c \
+		$(CONTRIBDIR)/uuid/unparse.c $(CONTRIBDIR)/uuid/uuid_time.c \
+		$(CONTRIBDIR)/uuid/compare.c $(CONTRIBDIR)/uuid/isnull.c \
+		$(CONTRIBDIR)/uuid/unpack.c
+
+noinst_HEADERS = gf-changelog-helpers.h $(CONTRIBDIR)/uuid/uuidd.h \
+		$(CONTRIBDIR)/uuid/uuid.h $(CONTRIBDIR)/uuid/uuidP.h \
+		$(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+
+libgfchangelog_HEADERS = changelog.h
+
+CLEANFILES =
+CONFIG_CLEAN_FILES = $(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+
+$(top_builddir)/libglusterfs/src/libglusterfs.la:
+	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
diff --git a/xlators/features/changelog/lib/src/changelog.h b/xlators/features/changelog/lib/src/changelog.h
new file mode 100644
index 000000000..5cddfb583
--- /dev/null
+++ b/xlators/features/changelog/lib/src/changelog.h
@@ -0,0 +1,31 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_H
+#define _GF_CHANGELOG_H
+
+/* API set */
+
+int
+gf_changelog_register (char *brick_path, char *scratch_dir,
+                       char *log_file, int log_levl, int max_reconnects);
+ssize_t
+gf_changelog_scan ();
+
+int
+gf_changelog_start_fresh ();
+
+ssize_t
+gf_changelog_next_change (char *bufptr, size_t maxlen);
+
+int
+gf_changelog_done (char *file);
+
+#endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.c b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
new file mode 100644
index 000000000..1eef8bf04
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
@@ -0,0 +1,180 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-mem-types.h"
+#include "gf-changelog-helpers.h"
+
+ssize_t gf_changelog_read_path (int fd, char *buffer, size_t bufsize)
+{
+        return read (fd, buffer, bufsize);
+}
+
+size_t
+gf_changelog_write (int fd, char *buffer, size_t len)
+{
+        ssize_t size = 0;
+        size_t writen = 0;
+
+        while (writen < len) {
+                size = write (fd,
+                              buffer + writen, len - writen);
+                if (size <= 0)
+                        break;
+
+                writen += size;
+        }
+
+        return writen;
+}
+
+void
+gf_rfc3986_encode (unsigned char *s, char *enc, char *estr)
+{
+        for (; *s; s++) {
+                if (estr[*s])
+                        sprintf(enc, "%c", estr[*s]);
+                else
+                        sprintf(enc, "%%%02X", *s);
+                while (*++enc);
+        }
+}
+
+/**
+ * thread safe version of readline with buffering
+ * (taken from Unix Network Programming Volume I, W.R. Stevens)
+ *
+ * This is favoured over fgets() as we'd need to ftruncate()
+ * (see gf_changelog_scan() API) to record new changelog files.
+ * stream open functions does have a truncate like api (although
+ * that can be done via @fflush(fp), @ftruncate(fd) and @fseek(fp),
+ * but this involves mixing POSIX file descriptors and stream FILE *).
+ *
+ * NOTE: This implmentation still does work with more than one fd's
+ *       used to perform gf_readline(). For this very reason it's not
+ *       made a part of libglusterfs.
+ */
+
+static pthread_key_t rl_key;
+static pthread_once_t rl_once = PTHREAD_ONCE_INIT;
+
+static void
+readline_destructor (void *ptr)
+{
+        GF_FREE (ptr);
+}
+
+static void
+readline_once (void)
+{
+        pthread_key_create (&rl_key, readline_destructor);
+}
+
+static ssize_t
+my_read (read_line_t *tsd, int fd, char *ptr)
+{
+        if (tsd->rl_cnt <= 0) {
+                if ( (tsd->rl_cnt = read (fd, tsd->rl_buf, MAXLINE)) < 0 )
+                        return -1;
+                else if (tsd->rl_cnt == 0)
+                        return 0;
+                tsd->rl_bufptr = tsd->rl_buf;
+        }
+
+        tsd->rl_cnt--;
+        *ptr = *tsd->rl_bufptr++;
+        return 1;
+}
+
+static int
+gf_readline_init_once (read_line_t **tsd)
+{
+        if (pthread_once (&rl_once, readline_once) != 0)
+                return -1;
+
+        *tsd = pthread_getspecific (rl_key);
+        if (*tsd)
+                goto out;
+
+        *tsd = GF_CALLOC (1, sizeof (**tsd),
+                          gf_changelog_mt_libgfchangelog_rl_t);
+        if (!*tsd)
+                return -1;
+
+        if (pthread_setspecific (rl_key, *tsd) != 0)
+                return -1;
+
+ out:
+        return 0;
+}
+
+ssize_t
+gf_readline (int fd, void *vptr, size_t maxlen)
+{
+        size_t       n   = 0;
+        size_t       rc  = 0;
+        char         c   = ' ';
+        char        *ptr = NULL;
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        ptr = vptr;
+        for (n = 1; n < maxlen; n++) {
+                if ( (rc = my_read (tsd, fd, &c)) == 1 ) {
+                        *ptr++ = c;
+                        if (c == '\n')
+                                break;
+                } else if (rc == 0) {
+                        *ptr = '\0';
+                        return (n - 1);
+                } else
+                        return -1;
+        }
+
+        *ptr = '\0';
+        return n;
+
+}
+
+off_t
+gf_lseek (int fd, off_t offset, int whence)
+{
+        off_t        off = 0;
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        if ( (off = lseek (fd, offset, whence)) == -1)
+                return -1;
+
+        tsd->rl_cnt = 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+
+        return off;
+}
+
+int
+gf_ftruncate (int fd, off_t length)
+{
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        if (ftruncate (fd, 0))
+                return -1;
+
+        tsd->rl_cnt = 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+
+        return 0;
+}
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.h b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
new file mode 100644
index 000000000..3aa6ed7b8
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
@@ -0,0 +1,97 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_HELPERS_H
+#define _GF_CHANGELOG_HELPERS_H
+
+#include <unistd.h>
+#include <dirent.h>
+#include <limits.h>
+#include <pthread.h>
+
+#include <xlator.h>
+
+#define GF_CHANGELOG_TRACKER  "tracker"
+
+#define GF_CHANGELOG_CURRENT_DIR    ".current"
+#define GF_CHANGELOG_PROCESSED_DIR  ".processed"
+#define GF_CHANGELOG_PROCESSING_DIR ".processing"
+
+#ifndef MAXLINE
+#define MAXLINE 4096
+#endif
+
+#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len) do {     \
+                memcpy (ascii + off, ptr, len);                 \
+                off += len;                                     \
+        } while (0)
+
+typedef struct read_line {
+        int rl_cnt;
+        char *rl_bufptr;
+        char rl_buf[MAXLINE];
+} read_line_t;
+
+typedef struct gf_changelog {
+        xlator_t *this;
+
+        /* 'processing' directory stream */
+        DIR *gfc_dir;
+
+        /* fd to the tracker file */
+        int gfc_fd;
+
+        /* connection retries */
+        int gfc_connretries;
+
+        char gfc_sockpath[PATH_MAX];
+
+        char gfc_brickpath[PATH_MAX];
+
+        /* socket for recieving notifications */
+        int gfc_sockfd;
+
+        char *gfc_working_dir;
+
+        /* RFC 3986 string encoding */
+        char rfc3986[256];
+
+        char gfc_current_dir[PATH_MAX];
+        char gfc_processed_dir[PATH_MAX];
+        char gfc_processing_dir[PATH_MAX];
+
+        pthread_t gfc_changelog_processor;
+} gf_changelog_t;
+
+int
+gf_changelog_notification_init (xlator_t *this, gf_changelog_t *gfc);
+
+void *
+gf_changelog_process (void *data);
+
+ssize_t
+gf_changelog_read_path (int fd, char *buffer, size_t bufsize);
+
+void
+gf_rfc3986_encode (unsigned char *s, char *enc, char *estr);
+
+size_t
+gf_changelog_write (int fd, char *buffer, size_t len);
+
+ssize_t
+gf_readline (int fd, void *vptr, size_t maxlen);
+
+int
+gf_ftruncate (int fd, off_t length);
+
+off_t
+gf_lseek (int fd, off_t offset, int whence);
+
+#endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-process.c b/xlators/features/changelog/lib/src/gf-changelog-process.c
new file mode 100644
index 000000000..df7204931
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-process.c
@@ -0,0 +1,571 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <pthread.h>
+
+#include "uuid.h"
+#include "globals.h"
+#include "glusterfs.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+
+extern int byebye;
+
+/**
+ * number of gfid records after fop number
+ */
+int nr_gfids[] = {
+        [GF_FOP_MKNOD]   = 1,
+        [GF_FOP_MKDIR]   = 1,
+        [GF_FOP_UNLINK]  = 1,
+        [GF_FOP_RMDIR]   = 1,
+        [GF_FOP_SYMLINK] = 1,
+        [GF_FOP_RENAME]  = 2,
+        [GF_FOP_LINK]    = 1,
+        [GF_FOP_CREATE]  = 1,
+};
+
+static char *
+binary_to_ascii (uuid_t uuid)
+{
+        return uuid_utoa (uuid);
+}
+
+static char *
+conv_noop (char *ptr) { return ptr; }
+
+#define VERIFY_SEPARATOR(ptr, plen, perr)       \
+        {                                       \
+                if (*(ptr + plen) != '\0') {    \
+                        perr = 1;               \
+                        break;                  \
+                }                               \
+        }
+
+#define MOVER_MOVE(mover, nleft, bytes)         \
+        {                                       \
+                mover += bytes;                 \
+                nleft -= bytes;                 \
+        }                                       \
+
+#define PARSE_GFID(mov, ptr, le, fn, perr)                      \
+        {                                                       \
+                VERIFY_SEPARATOR (mov, le, perr);               \
+                ptr = fn (mov);                                 \
+                if (!ptr) {                                     \
+                        perr = 1;                               \
+                        break;                                  \
+                }                                               \
+        }
+
+#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                          \
+        {                                                               \
+                GF_CHANGELOG_FILL_BUFFER (pt, buf, of, strlen (pt));    \
+                MOVER_MOVE (mo, nl, le);                                \
+        }
+
+
+#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)          \
+        {                                                       \
+                memcpy (uuid, mover, sizeof (uuid_t));          \
+                ptr = binary_to_ascii (uuid);                   \
+                if (!ptr) {                                     \
+                        perr = 1;                               \
+                        break;                                  \
+                }                                               \
+                MOVER_MOVE (mover, nleft, sizeof (uuid_t));     \
+        }                                                       \
+
+#define LINE_BUFSIZE  3*PATH_MAX /* enough buffer for extra chars too */
+
+/**
+ * using mmap() makes parsing easy. fgets() cannot be used here as
+ * the binary gfid could contain a line-feed (0x0A), in that case fgets()
+ * would read an incomplete line and parsing would fail. using POSIX fds
+ * would result is additional code to maintain state in case of partial
+ * reads of data (where multiple entries do not fit extirely in the buffer).
+ *
+ * mmap() gives the flexibility of pointing to an offset in the file
+ * without us worrying about reading it in memory (VM does that for us for
+ * free).
+ */
+
+static int
+gf_changelog_parse_binary (xlator_t *this,
+                           gf_changelog_t *gfc, int from_fd, int to_fd,
+                           size_t start_offset, struct stat *stbuf)
+
+{
+        int     ret              = -1;
+        off_t   off              = 0;
+        off_t   nleft            = 0;
+        uuid_t  uuid             = {0,};
+        char   *ptr              = NULL;
+        char   *bname_start      = NULL;
+        char   *bname_end        = NULL;
+        char   *mover            = NULL;
+        char   *start            = NULL;
+        char    current_mover    = ' ';
+        size_t  blen             = 0;
+        int     parse_err        = 0;
+        char ascii[LINE_BUFSIZE] = {0,};
+
+        nleft = stbuf->st_size;
+
+        start = (char *) mmap (NULL, nleft,
+                               PROT_READ, MAP_PRIVATE, from_fd, 0);
+        if (!start) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "mmap() error (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        mover = start;
+
+        MOVER_MOVE (mover, nleft, start_offset);
+
+        while (nleft > 0) {
+
+                off = blen = 0;
+                ptr = bname_start = bname_end = NULL;
+
+                current_mover = *mover;
+
+                switch (current_mover) {
+                case 'D':
+                case 'M':
+                        MOVER_MOVE (mover, nleft, 1);
+                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+
+                        break;
+
+                case 'E':
+                        MOVER_MOVE (mover, nleft, 1);
+                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+
+                        bname_start = mover;
+                        if ( (bname_end = strchr (mover, '\n')) == NULL ) {
+                                parse_err = 1;
+                                break;
+                        }
+
+                        blen = bname_end - bname_start;
+                        MOVER_MOVE (mover, nleft, blen);
+
+                        break;
+
+                default:
+                        parse_err = 1;
+                }
+
+                if (parse_err)
+                        break;
+
+                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (ptr, ascii, off, strlen (ptr));
+                if (blen)
+                        GF_CHANGELOG_FILL_BUFFER (bname_start,
+                                                  ascii, off, blen);
+                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
+
+                if (gf_changelog_write (to_fd, ascii, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "processing binary changelog failed due to "
+                                " error in writing ascii change (reason: %s)",
+                                strerror (errno));
+                        break;
+                }
+
+                MOVER_MOVE (mover, nleft, 1);
+        }
+
+        if ( (nleft == 0) && (!parse_err))
+                ret = 0;
+
+        if (munmap (start, stbuf->st_size))
+                gf_log (this->name, GF_LOG_ERROR,
+                        "munmap() error (reason: %s)", strerror (errno));
+ out:
+        return ret;
+}
+
+/**
+ * ascii decoder:
+ *  - separate out one entry from another
+ *  - use fop name rather than fop number
+ */
+static int
+gf_changelog_parse_ascii (xlator_t *this,
+                          gf_changelog_t *gfc, int from_fd, int to_fd,
+                          size_t start_offset, struct stat *stbuf)
+{
+        int         ng            = 0;
+        int         ret           = -1;
+        int         fop           = 0;
+        int         len           = 0;
+        off_t       off           = 0;
+        off_t       nleft         = 0;
+        char       *ptr           = NULL;
+        char       *eptr          = NULL;
+        char       *start         = NULL;
+        char       *mover         = NULL;
+        int         parse_err     = 0;
+        char        current_mover = ' ';
+        char ascii[LINE_BUFSIZE]  = {0,};
+        const char *fopname       = NULL;
+
+        nleft = stbuf->st_size;
+
+        start = (char *) mmap (NULL, nleft,
+                               PROT_READ, MAP_PRIVATE, from_fd, 0);
+        if (!start) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "mmap() error (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        mover = start;
+
+        MOVER_MOVE (mover, nleft, start_offset);
+
+        while (nleft > 0) {
+                off = 0;
+                current_mover = *mover;
+
+                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+
+                switch (current_mover) {
+                case 'D':
+                case 'M':
+                        MOVER_MOVE (mover, nleft, 1);
+
+                        /* target gfid */
+                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
+                                    conv_noop, parse_err);
+                        FILL_AND_MOVE(ptr, ascii, off,
+                                      mover, nleft, UUID_CANONICAL_FORM_LEN);
+                        break;
+
+                case 'E':
+                        MOVER_MOVE (mover, nleft, 1);
+
+                        /* target gfid */
+                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
+                                    conv_noop, parse_err);
+                        FILL_AND_MOVE (ptr, ascii, off,
+                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
+                        FILL_AND_MOVE (" ", ascii, off,
+                                       mover, nleft, 1);
+
+                        /* fop */
+                        len = strlen (mover);
+                        VERIFY_SEPARATOR (mover, len, parse_err);
+
+                        fop = atoi (mover);
+                        if ( (fopname = gf_fop_list[fop]) == NULL) {
+                                parse_err = 1;
+                                break;
+                        }
+
+                        MOVER_MOVE (mover, nleft, len);
+
+                        len = strlen (fopname);
+                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
+
+                        /* pargfid + bname */
+                        ng = nr_gfids[fop];
+                        while (ng-- > 0) {
+                                MOVER_MOVE (mover, nleft, 1);
+                                len = strlen (mover);
+                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+
+                                PARSE_GFID (mover, ptr, len,
+                                            conv_noop, parse_err);
+                                eptr = calloc (3, strlen (ptr));
+                                if (!eptr) {
+                                        parse_err = 1;
+                                        break;
+                                }
+
+                                gf_rfc3986_encode ((unsigned char *) ptr,
+                                                   eptr, gfc->rfc3986);
+                                FILL_AND_MOVE (eptr, ascii, off,
+                                               mover, nleft, len);
+                                free (eptr);
+                        }
+
+                        break;
+                default:
+                        parse_err = 1;
+                }
+
+                if (parse_err)
+                        break;
+
+                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
+
+                if (gf_changelog_write (to_fd, ascii, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "processing ascii changelog failed due to "
+                                " wrror in writing change (reason: %s)",
+                                strerror (errno));
+                        break;
+                }
+
+                MOVER_MOVE (mover, nleft, 1);
+
+        }
+
+        if ( (nleft == 0) && (!parse_err))
+                ret = 0;
+
+        if (munmap (start, stbuf->st_size))
+                gf_log (this->name, GF_LOG_ERROR,
+                        "munmap() error (reason: %s)", strerror (errno));
+
+ out:
+        return ret;
+}
+
+#define COPY_BUFSIZE  8192
+static int
+gf_changelog_copy (xlator_t *this, int from_fd, int to_fd)
+{
+        ssize_t size                  = 0;
+        char   buffer[COPY_BUFSIZE+1] = {0,};
+
+        while (1) {
+                size = read (from_fd, buffer, COPY_BUFSIZE);
+                if (size <= 0)
+                        break;
+
+                if (gf_changelog_write (to_fd,
+                                        buffer, size) != size) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error processing ascii changlog");
+                        size = -1;
+                        break;
+                }
+        }
+
+        return (size < 0 ? -1 : 0);
+}
+
+static int
+gf_changelog_decode (xlator_t *this, gf_changelog_t *gfc, int from_fd,
+                     int to_fd, struct stat *stbuf, int *zerob)
+{
+        int    ret        = -1;
+        int    encoding   = -1;
+        size_t elen       = 0;
+        char buffer[1024] = {0,};
+
+        CHANGELOG_GET_ENCODING (from_fd, buffer, 1024, encoding, elen);
+        if (encoding == -1) /* unknown encoding */
+                goto out;
+
+        if (!CHANGELOG_VALID_ENCODING (encoding))
+                goto out;
+
+        if (elen == stbuf->st_size) {
+                *zerob = 1;
+                goto out;
+        }
+
+        /**
+         * start processing after the header
+         */
+        lseek (from_fd, elen, SEEK_SET);
+
+        switch (encoding) {
+        case CHANGELOG_ENCODE_BINARY:
+                /**
+                 * this ideally should have been a part of changelog-encoders.c
+                 * (ie. part of the changelog translator).
+                 */
+                ret = gf_changelog_parse_binary (this, gfc, from_fd,
+                                                 to_fd, elen, stbuf);
+                break;
+
+        case CHANGELOG_ENCODE_ASCII:
+                ret = gf_changelog_parse_ascii (this, gfc, from_fd,
+                                                to_fd, elen, stbuf);
+                break;
+        default:
+                ret = gf_changelog_copy (this, from_fd, to_fd);
+        }
+
+ out:
+        return ret;
+}
+
+static int
+gf_changelog_consume (xlator_t *this, gf_changelog_t *gfc, char *from_path)
+{
+        int         ret        = -1;
+        int         fd1        = 0;
+        int         fd2        = 0;
+        int         zerob      = 0;
+        struct stat stbuf      = {0,};
+        char dest[PATH_MAX]    = {0,};
+        char to_path[PATH_MAX] = {0,};
+
+        ret = stat (from_path, &stbuf);
+        if (ret || !S_ISREG(stbuf.st_mode)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat failed on changelog file: %s", from_path);
+                goto out;
+        }
+
+        fd1 = open (from_path, O_RDONLY);
+        if (fd1 < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot open changelog file: %s (reason: %s)",
+                        from_path, strerror (errno));
+                goto out;
+        }
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         gfc->gfc_current_dir, basename (from_path));
+        (void) snprintf (dest, PATH_MAX, "%s%s",
+                         gfc->gfc_processing_dir, basename (from_path));
+
+        fd2 = open (to_path, O_CREAT | O_TRUNC | O_RDWR,
+                    S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (fd2 < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot create ascii changelog file %s (reason %s)",
+                        to_path, strerror (errno));
+                goto close_fd;
+        } else {
+                ret = gf_changelog_decode (this, gfc, fd1,
+                                           fd2, &stbuf, &zerob);
+
+                close (fd2);
+
+                if (!ret) {
+                        /* move it to processing on a successfull
+                           decode */
+                        ret = rename (to_path, dest);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "error moving %s to processing dir"
+                                        " (reason: %s)", to_path,
+                                        strerror (errno));
+                }
+
+                /* remove it from .current if it's an empty file */
+                if (zerob) {
+                        ret = unlink (to_path);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not unlink %s (reason: %s",
+                                        to_path, strerror (errno));
+                }
+        }
+
+ close_fd:
+        close (fd1);
+
+ out:
+        return ret;
+}
+
+static char *
+gf_changelog_ext_change (xlator_t *this,
+                         gf_changelog_t *gfc, char *path, size_t readlen)
+{
+        int     alo = 0;
+        int     ret = 0;
+        size_t  len = 0;
+        char   *buf = NULL;
+
+        buf = path;
+        while (len < readlen) {
+                if (*buf == '\0') {
+                        alo = 1;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "processing changelog: %s", path);
+                        ret = gf_changelog_consume (this, gfc, path);
+                }
+
+                if (ret)
+                        break;
+
+                len++; buf++;
+                if (alo) {
+                        alo = 0;
+                        path = buf;
+                }
+        }
+
+        return (ret) ? NULL : path;
+}
+
+void *
+gf_changelog_process (void *data)
+{
+        ssize_t         len      = 0;
+        ssize_t         offlen   = 0;
+        xlator_t       *this     = NULL;
+        char           *sbuf     = NULL;
+        gf_changelog_t *gfc      = NULL;
+        char from_path[PATH_MAX] = {0,};
+
+        gfc = (gf_changelog_t *) data;
+        this = gfc->this;
+
+        pthread_detach (pthread_self());
+
+        for (;;) {
+                len = gf_changelog_read_path (gfc->gfc_sockfd,
+                                              from_path + offlen,
+                                              PATH_MAX - offlen);
+                if (len < 0)
+                        continue; /* ignore it for now */
+
+                if (len == 0) { /* close() from the changelog translator */
+                        gf_log (this->name, GF_LOG_INFO, "close from changelog"
+                                " notification translator.");
+
+                        if (gfc->gfc_connretries != 1) {
+                                if (!gf_changelog_notification_init(this, gfc))
+                                        continue;
+                        }
+
+                        byebye = 1;
+                        break;
+                }
+
+                len += offlen;
+                sbuf = gf_changelog_ext_change (this, gfc, from_path, len);
+                if (!sbuf) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not extract changelog filename");
+                        continue;
+                }
+
+                offlen = 0;
+                if (sbuf != (from_path + len)) {
+                        offlen = from_path + len - sbuf;
+                        memmove (from_path, sbuf, offlen);
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "byebye (%d) from processing thread...", byebye);
+        return NULL;
+}
diff --git a/xlators/features/changelog/lib/src/gf-changelog.c b/xlators/features/changelog/lib/src/gf-changelog.c
new file mode 100644
index 000000000..ca8e373e7
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog.c
@@ -0,0 +1,515 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+#include <dirent.h>
+#include <stddef.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+#include "changelog-mem-types.h"
+
+int byebye = 0;
+
+static void
+gf_changelog_cleanup (gf_changelog_t *gfc)
+{
+        /* socket */
+        if (gfc->gfc_sockfd != -1)
+                close (gfc->gfc_sockfd);
+        /* tracker fd */
+        if (gfc->gfc_fd != -1)
+                close (gfc->gfc_fd);
+        /* processing dir */
+        if (gfc->gfc_dir)
+                closedir (gfc->gfc_dir);
+
+        if (gfc->gfc_working_dir)
+                free (gfc->gfc_working_dir); /* allocated by realpath */
+}
+
+void
+__attribute__ ((constructor)) gf_changelog_ctor (void)
+{
+        glusterfs_ctx_t *ctx = NULL;
+
+        ctx = glusterfs_ctx_new ();
+        if (!ctx)
+                return;
+
+        if (glusterfs_globals_init (ctx)) {
+                free (ctx);
+                ctx = NULL;
+                return;
+        }
+
+        THIS->ctx = ctx;
+}
+
+void
+__attribute__ ((destructor)) gf_changelog_dtor (void)
+{
+        xlator_t        *this = NULL;
+        glusterfs_ctx_t *ctx  = NULL;
+        gf_changelog_t  *gfc  = NULL;
+
+        this = THIS;
+        if (!this)
+                return;
+
+        ctx = this->ctx;
+        gfc = this->private;
+
+        if (gfc) {
+                gf_changelog_cleanup (gfc);
+                GF_FREE (gfc);
+        }
+
+        if (ctx) {
+                pthread_mutex_destroy (&ctx->lock);
+                free (ctx);
+                ctx = NULL;
+        }
+}
+
+
+static int
+gf_changelog_open_dirs (gf_changelog_t *gfc)
+{
+        int  ret                    = -1;
+        DIR *dir                    = NULL;
+        int  tracker_fd             = 0;
+        char tracker_path[PATH_MAX] = {0,};
+
+        (void) snprintf (gfc->gfc_current_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_CURRENT_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_current_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        (void) snprintf (gfc->gfc_processed_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_PROCESSED_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_processed_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        (void) snprintf (gfc->gfc_processing_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_PROCESSING_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_processing_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        dir = opendir (gfc->gfc_processing_dir);
+        if (!dir) {
+                gf_log ("", GF_LOG_ERROR,
+                        "opendir() error [reason: %s]", strerror (errno));
+                goto out;
+        }
+
+        gfc->gfc_dir = dir;
+
+        (void) snprintf (tracker_path, PATH_MAX,
+                         "%s/"GF_CHANGELOG_TRACKER, gfc->gfc_working_dir);
+
+        tracker_fd = open (tracker_path, O_CREAT | O_APPEND | O_RDWR,
+                           S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (tracker_fd < 0) {
+                closedir (gfc->gfc_dir);
+                ret = -1;
+                goto out;
+        }
+
+        gfc->gfc_fd = tracker_fd;
+        ret = 0;
+ out:
+        return ret;
+}
+
+int
+gf_changelog_notification_init (xlator_t *this, gf_changelog_t *gfc)
+{
+        int                ret    = 0;
+        int                len    = 0;
+        int                tries  = 0;
+        int                sockfd = 0;
+        struct sockaddr_un remote;
+
+        this = gfc->this;
+
+        if (gfc->gfc_sockfd != -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Reconnecting...");
+                close (gfc->gfc_sockfd);
+        }
+
+        sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
+        if (sockfd < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        CHANGELOG_MAKE_SOCKET_PATH (gfc->gfc_brickpath,
+                                    gfc->gfc_sockpath, PATH_MAX);
+        gf_log (this->name, GF_LOG_INFO,
+                "connecting to changelog socket: %s (brick: %s)",
+                gfc->gfc_sockpath, gfc->gfc_brickpath);
+
+        remote.sun_family = AF_UNIX;
+        strcpy (remote.sun_path, gfc->gfc_sockpath);
+
+        len = strlen (remote.sun_path) + sizeof (remote.sun_family);
+
+        while (tries < gfc->gfc_connretries) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "connection attempt %d/%d...",
+                        tries + 1, gfc->gfc_connretries);
+
+                /* initiate a connect */
+                if (connect (sockfd, (struct sockaddr *) &remote, len) == 0) {
+                        gfc->gfc_sockfd = sockfd;
+                        break;
+                }
+
+                tries++;
+                sleep (2);
+        }
+
+        if (tries == gfc->gfc_connretries) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not connect to changelog socket!"
+                        " bailing out...");
+                ret = -1;
+        } else
+                gf_log (this->name, GF_LOG_INFO,
+                        "connection successful");
+
+ out:
+        return ret;
+}
+
+int
+gf_changelog_done (char *file)
+{
+        int             ret    = -1;
+        char           *buffer = NULL;
+        xlator_t       *this   = NULL;
+        gf_changelog_t *gfc    = NULL;
+        char to_path[PATH_MAX] = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        if (!file || !strlen (file))
+                goto out;
+
+        /* make sure 'file' is inside ->gfc_working_dir */
+        buffer = realpath (file, NULL);
+        if (!buffer)
+                goto out;
+
+        if (strncmp (gfc->gfc_working_dir,
+                     buffer, strlen (gfc->gfc_working_dir)))
+                goto out;
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         gfc->gfc_processed_dir, basename (buffer));
+        gf_log (this->name, GF_LOG_DEBUG,
+                "moving %s to processed directory", file);
+        ret = rename (buffer, to_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot move %s to %s (reason: %s)",
+                        file, to_path, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        if (buffer)
+                free (buffer); /* allocated by realpath() */
+        return ret;
+}
+
+/**
+ * @API
+ *  for a set of changelogs, start from the begining
+ */
+int
+gf_changelog_start_fresh ()
+{
+        xlator_t *this = NULL;
+        gf_changelog_t *gfc = NULL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        errno = EINVAL;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        if (gf_ftruncate (gfc->gfc_fd, 0))
+                goto out;
+
+        return 0;
+
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ * return the next changelog file entry. zero means all chanelogs
+ * consumed.
+ */
+ssize_t
+gf_changelog_next_change (char *bufptr, size_t maxlen)
+{
+        ssize_t         size       = 0;
+        int             tracker_fd = 0;
+        xlator_t       *this       = NULL;
+        gf_changelog_t *gfc        = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        tracker_fd = gfc->gfc_fd;
+
+        size = gf_readline (tracker_fd, buffer, maxlen);
+        if (size < 0)
+                goto out;
+        if (size == 0)
+                return 0;
+
+        memcpy (bufptr, buffer, size - 1);
+        *(buffer + size) = '\0';
+
+        return size;
+
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ *  gf_changelog_scan() - scan and generate a list of change entries
+ *
+ * calling this api multiple times (without calling gf_changlog_done())
+ * would result new changelogs(s) being refreshed in the tracker file.
+ * This call also acts as a cancellation point for the consumer.
+ */
+ssize_t
+gf_changelog_scan ()
+{
+        int             ret        = 0;
+        int             tracker_fd = 0;
+        size_t          len        = 0;
+        size_t          off        = 0;
+        xlator_t       *this       = NULL;
+        size_t          nr_entries = 0;
+        gf_changelog_t *gfc        = NULL;
+        struct dirent  *entryp     = NULL;
+        struct dirent  *result     = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        /**
+         * do we need to protect 'byebye' with locks? worst, the
+         * consumer would get notified during next scan().
+         */
+        if (byebye) {
+                errno = ECONNREFUSED;
+                goto out;
+        }
+
+        errno = EINVAL;
+
+        tracker_fd = gfc->gfc_fd;
+
+        if (gf_ftruncate (tracker_fd, 0))
+                goto out;
+
+        len = offsetof(struct dirent, d_name)
+                + pathconf(gfc->gfc_processing_dir, _PC_NAME_MAX) + 1;
+        entryp = GF_CALLOC (1, len,
+                            gf_changelog_mt_libgfchangelog_dirent_t);
+        if (!entryp)
+                goto out;
+
+        rewinddir (gfc->gfc_dir);
+        while (1) {
+                ret = readdir_r (gfc->gfc_dir, entryp, &result);
+                if (ret || !result)
+                        break;
+
+                if ( !strcmp (basename (entryp->d_name), ".")
+                     || !strcmp (basename (entryp->d_name), "..") )
+                        continue;
+
+                nr_entries++;
+
+                GF_CHANGELOG_FILL_BUFFER (gfc->gfc_processing_dir,
+                                          buffer, off,
+                                          strlen (gfc->gfc_processing_dir));
+                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
+                                          off, strlen (entryp->d_name));
+                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
+
+                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error writing changelog filename"
+                                " to tracker file");
+                        break;
+                }
+                off = 0;
+        }
+
+        GF_FREE (entryp);
+
+        if (!result) {
+                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1)
+                        return nr_entries;
+        }
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ *  gf_changelog_register() - register a client for updates.
+ */
+int
+gf_changelog_register (char *brick_path, char *scratch_dir,
+                       char *log_file, int log_level, int max_reconnects)
+{
+        int             i    = 0;
+        int             ret  = -1;
+        int             errn = 0;
+        xlator_t       *this = NULL;
+        gf_changelog_t *gfc  = NULL;
+
+        this = THIS;
+        if (!this->ctx)
+                goto out;
+
+        errno = ENOMEM;
+
+        gfc = GF_CALLOC (1, sizeof (*gfc),
+                         gf_changelog_mt_libgfchangelog_t);
+        if (!gfc)
+                goto out;
+
+        gfc->this = this;
+
+        gfc->gfc_dir = NULL;
+        gfc->gfc_fd = gfc->gfc_sockfd = -1;
+
+        gfc->gfc_working_dir = realpath (scratch_dir, NULL);
+        if (!gfc->gfc_working_dir) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        ret = gf_changelog_open_dirs (gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not create entries in scratch dir");
+                goto cleanup;
+        }
+
+        /* passing ident as NULL means to use default ident for syslog */
+        if (gf_log_init (this->ctx, log_file, NULL))
+                goto cleanup;
+
+        gf_log_set_loglevel ((log_level == -1) ? GF_LOG_INFO :
+                             log_level);
+
+        gfc->gfc_connretries = (max_reconnects <= 0) ? 1 : max_reconnects;
+        (void) strncpy (gfc->gfc_brickpath, brick_path, PATH_MAX);
+
+        ret = gf_changelog_notification_init (this, gfc);
+        if (ret) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        ret = gf_thread_create (&gfc->gfc_changelog_processor,
+				NULL, gf_changelog_process, gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error creating changelog processor thread"
+                        " new changes won't be recorded!!!");
+                goto cleanup;
+        }
+
+        for (; i < 256; i++) {
+                gfc->rfc3986[i] =
+                        (isalnum(i) || i == '~' ||
+                        i == '-' || i == '.' || i == '_') ? i : 0;
+        }
+
+        ret = 0;
+        this->private = gfc;
+
+        goto out;
+
+ cleanup:
+        gf_changelog_cleanup (gfc);
+        GF_FREE (gfc);
+        this->private = NULL;
+        errno = errn;
+
+ out:
+        return ret;
+}
diff --git a/xlators/features/changelog/src/Makefile.am b/xlators/features/changelog/src/Makefile.am
new file mode 100644
index 000000000..e85031ad4
--- /dev/null
+++ b/xlators/features/changelog/src/Makefile.am
@@ -0,0 +1,19 @@
+xlator_LTLIBRARIES = changelog.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = changelog-helpers.h changelog-mem-types.h changelog-rt.h \
+	changelog-misc.h changelog-encoders.h changelog-notifier.h
+
+changelog_la_LDFLAGS = -module -avoidversion
+
+changelog_la_SOURCES = changelog.c changelog-rt.c changelog-helpers.c \
+	changelog-encoders.c changelog-notifier.c
+changelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC -D_FILE_OFFSET_BITS=64 \
+	-D_GNU_SOURCE -D$(GF_HOST_OS) -shared -nostartfiles -DDATADIR=\"$(localstatedir)\"
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/changelog/src/changelog-encoders.c b/xlators/features/changelog/src/changelog-encoders.c
new file mode 100644
index 000000000..553eec85c
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-encoders.c
@@ -0,0 +1,176 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "changelog-encoders.h"
+
+size_t
+entry_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char    *tmpbuf = NULL;
+        size_t  bufsz  = 0;
+        struct changelog_entry_fields *ce = NULL;
+
+        ce = (struct changelog_entry_fields *) data;
+
+        if (encode) {
+                tmpbuf = uuid_utoa (ce->cef_uuid);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, tmpbuf, strlen (tmpbuf));
+        } else {
+                CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                                       ce->cef_uuid, sizeof (uuid_t));
+        }
+
+        CHANGELOG_FILL_BUFFER (buffer, bufsz, "/", 1);
+        CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                               ce->cef_bname, strlen (ce->cef_bname));
+        return bufsz;
+}
+
+size_t
+fop_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char buf[10]          = {0,};
+        size_t         bufsz = 0;
+        glusterfs_fop_t fop   = 0;
+
+        fop = *(glusterfs_fop_t *) data;
+
+        if (encode) {
+                (void) snprintf (buf, sizeof (buf), "%d", fop);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
+        } else
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, &fop, sizeof (fop));
+
+        return bufsz;
+}
+
+void
+entry_free_fn (void *data)
+{
+        changelog_opt_t *co = data;
+
+        if (!co)
+                return;
+
+        GF_FREE (co->co_entry.cef_bname);
+}
+
+/**
+ * try to write all data in one shot
+ */
+
+static inline void
+changelog_encode_write_xtra (changelog_log_data_t *cld,
+                             char *buffer, size_t *off, gf_boolean_t encode)
+{
+        int              i      = 0;
+        size_t           offset = 0;
+        void            *data   = NULL;
+        changelog_opt_t *co     = NULL;
+
+        offset = *off;
+
+        co = (changelog_opt_t *) cld->cld_ptr;
+
+        for (; i < cld->cld_xtra_records; i++, co++) {
+                CHANGELOG_FILL_BUFFER (buffer, offset, "\0", 1);
+
+                switch (co->co_type) {
+                case CHANGELOG_OPT_REC_FOP:
+                        data = &co->co_fop;
+                        break;
+                case CHANGELOG_OPT_REC_ENTRY:
+                        data = &co->co_entry;
+                        break;
+                }
+
+                if (co->co_convert)
+                        offset += co->co_convert (data,
+                                                  buffer + offset, encode);
+                else /* no coversion: write it out as it is */
+                        CHANGELOG_FILL_BUFFER (buffer, offset,
+                                               data, co->co_len);
+        }
+
+        *off = offset;
+}
+
+int
+changelog_encode_ascii (xlator_t *this, changelog_log_data_t *cld)
+{
+        size_t            off      = 0;
+        size_t            gfid_len = 0;
+        char             *gfid_str = NULL;
+        char             *buffer   = NULL;
+        changelog_priv_t *priv     = NULL;
+
+        priv = this->private;
+
+        gfid_str = uuid_utoa (cld->cld_gfid);
+        gfid_len = strlen (gfid_str);
+
+        /* extra bytes for decorations */
+        buffer = alloca (gfid_len + cld->cld_ptr_len + 10);
+        CHANGELOG_STORE_ASCII (priv, buffer,
+                               off, gfid_str, gfid_len, cld);
+
+        if (cld->cld_xtra_records)
+                changelog_encode_write_xtra (cld, buffer, &off, _gf_true);
+
+        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+
+        return changelog_write_change (priv, buffer, off);
+}
+
+int
+changelog_encode_binary (xlator_t *this, changelog_log_data_t *cld)
+{
+        size_t            off    = 0;
+        char             *buffer = NULL;
+        changelog_priv_t *priv   = NULL;
+
+        priv = this->private;
+
+        /* extra bytes for decorations */
+        buffer = alloca (sizeof (uuid_t) + cld->cld_ptr_len + 10);
+        CHANGELOG_STORE_BINARY (priv, buffer, off, cld->cld_gfid, cld);
+
+        if (cld->cld_xtra_records)
+                changelog_encode_write_xtra (cld, buffer, &off, _gf_false);
+
+        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+
+        return changelog_write_change (priv, buffer, off);
+}
+
+static struct changelog_encoder
+cb_encoder[] = {
+        [CHANGELOG_ENCODE_BINARY] =
+        {
+                .encoder = CHANGELOG_ENCODE_BINARY,
+                .encode = changelog_encode_binary,
+        },
+        [CHANGELOG_ENCODE_ASCII] =
+        {
+                .encoder = CHANGELOG_ENCODE_ASCII,
+                .encode = changelog_encode_ascii,
+        },
+};
+
+void
+changelog_encode_change( changelog_priv_t * priv)
+{
+        priv->ce = &cb_encoder[priv->encode_mode];
+}
diff --git a/xlators/features/changelog/src/changelog-encoders.h b/xlators/features/changelog/src/changelog-encoders.h
new file mode 100644
index 000000000..a3efbee05
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-encoders.h
@@ -0,0 +1,46 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_ENCODERS_H
+#define _CHANGELOG_ENCODERS_H
+
+#include "xlator.h"
+#include "defaults.h"
+
+#include "changelog-helpers.h"
+
+#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld) do { \
+                CHANGELOG_FILL_BUFFER (buffer, off,                     \
+                                       priv->maps[cld->cld_type], 1);   \
+                CHANGELOG_FILL_BUFFER (buffer,                          \
+                                       off, gfid, gfid_len);            \
+        } while (0)
+
+#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld) do {          \
+                CHANGELOG_FILL_BUFFER (buffer, off,                     \
+                                       priv->maps[cld->cld_type], 1);   \
+                CHANGELOG_FILL_BUFFER (buffer,                          \
+                                       off, gfid, sizeof (uuid_t));     \
+        } while (0)
+
+size_t
+entry_fn (void *data, char *buffer, gf_boolean_t encode);
+size_t
+fop_fn (void *data, char *buffer, gf_boolean_t encode);
+void
+entry_free_fn (void *data);
+int
+changelog_encode_binary (xlator_t *, changelog_log_data_t *);
+int
+changelog_encode_ascii (xlator_t *, changelog_log_data_t *);
+void
+changelog_encode_change(changelog_priv_t *);
+
+#endif /* _CHANGELOG_ENCODERS_H */
diff --git a/xlators/features/changelog/src/changelog-helpers.c b/xlators/features/changelog/src/changelog-helpers.c
new file mode 100644
index 000000000..7ab0091b5
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-helpers.c
@@ -0,0 +1,693 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+#include "iobuf.h"
+
+#include "changelog-helpers.h"
+#include "changelog-mem-types.h"
+
+#include "changelog-encoders.h"
+#include <pthread.h>
+
+void
+changelog_thread_cleanup (xlator_t *this, pthread_t thr_id)
+{
+        int   ret    = 0;
+        void *retval = NULL;
+
+        /* send a cancel request to the thread */
+        ret = pthread_cancel (thr_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not cancel thread (reason: %s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        ret = pthread_join (thr_id, &retval);
+        if (ret || (retval != PTHREAD_CANCELED)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cancel request not adhered as expected"
+                        " (reason: %s)", strerror (errno));
+        }
+
+ out:
+        return;
+}
+
+inline void *
+changelog_get_usable_buffer (changelog_local_t *local)
+{
+        changelog_log_data_t *cld = NULL;
+
+        cld = &local->cld;
+        if (!cld->cld_iobuf)
+                return NULL;
+
+        return cld->cld_iobuf->ptr;
+}
+
+inline void
+changelog_set_usable_record_and_length (changelog_local_t *local,
+                                        size_t len, int xr)
+{
+        changelog_log_data_t *cld = NULL;
+
+        cld = &local->cld;
+
+        cld->cld_ptr_len = len;
+        cld->cld_xtra_records = xr;
+}
+
+void
+changelog_local_cleanup (xlator_t *xl, changelog_local_t *local)
+{
+        int                   i   = 0;
+        changelog_opt_t      *co  = NULL;
+        changelog_log_data_t *cld = NULL;
+
+        if (!local)
+                return;
+
+        cld = &local->cld;
+
+        /* cleanup dynamic allocation for extra records */
+        if (cld->cld_xtra_records) {
+                co = (changelog_opt_t *) cld->cld_ptr;
+                for (; i < cld->cld_xtra_records; i++, co++)
+                        if (co->co_free)
+                                co->co_free (co);
+        }
+
+        CHANGELOG_IOBUF_UNREF (cld->cld_iobuf);
+
+        if (local->inode)
+                inode_unref (local->inode);
+
+        mem_put (local);
+}
+
+inline int
+changelog_write (int fd, char *buffer, size_t len)
+{
+        ssize_t size = 0;
+        size_t writen = 0;
+
+        while (writen < len) {
+                size = write (fd,
+                              buffer + writen, len - writen);
+                if (size <= 0)
+                        break;
+
+                writen += size;
+        }
+
+        return (writen != len);
+}
+
+static int
+changelog_rollover_changelog (xlator_t *this,
+                              changelog_priv_t *priv, unsigned long ts)
+{
+        int   ret            = -1;
+        int   notify         = 0;
+        char *bname          = NULL;
+        char ofile[PATH_MAX] = {0,};
+        char nfile[PATH_MAX] = {0,};
+
+        if (priv->changelog_fd != -1) {
+                close (priv->changelog_fd);
+                priv->changelog_fd = -1;
+        }
+
+        (void) snprintf (ofile, PATH_MAX,
+                         "%s/"CHANGELOG_FILE_NAME, priv->changelog_dir);
+        (void) snprintf (nfile, PATH_MAX,
+                         "%s/"CHANGELOG_FILE_NAME".%lu",
+                         priv->changelog_dir, ts);
+
+        ret = rename (ofile, nfile);
+        if (!ret)
+                notify = 1;
+
+        if (ret && (errno == ENOENT)) {
+                ret = 0;
+        }
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error renaming %s -> %s (reason %s)",
+                        ofile, nfile, strerror (errno));
+        }
+
+        if (notify) {
+                bname = basename (nfile);
+                gf_log (this->name, GF_LOG_DEBUG, "notifying: %s", bname);
+                ret = changelog_write (priv->wfd, bname, strlen (bname) + 1);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to send file name to notify thread"
+                                " (reason: %s)", strerror (errno));
+                }
+        }
+
+        return ret;
+}
+
+int
+changelog_open (xlator_t *this,
+                changelog_priv_t *priv)
+{
+        int fd                        = 0;
+        int ret                       = -1;
+        int flags                     = 0;
+        char buffer[1024]             = {0,};
+        char changelog_path[PATH_MAX] = {0,};
+
+        (void) snprintf (changelog_path, PATH_MAX,
+                         "%s/"CHANGELOG_FILE_NAME,
+                         priv->changelog_dir);
+
+        flags |= (O_CREAT | O_RDWR);
+        if (priv->fsync_interval == 0)
+                flags |= O_SYNC;
+
+        fd = open (changelog_path, flags,
+                   S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to open/create changelog file %s"
+                        " (reason: %s). change-logging will be"
+                        " inactive", changelog_path, strerror (errno));
+                goto out;
+        }
+
+        priv->changelog_fd = fd;
+
+        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
+                         CHANGELOG_VERSION_MAJOR,
+                         CHANGELOG_VERSION_MINOR,
+                         priv->ce->encoder);
+        ret = changelog_write_change (priv, buffer, strlen (buffer));
+        if (ret) {
+                close (priv->changelog_fd);
+                priv->changelog_fd = -1;
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int
+changelog_start_next_change (xlator_t *this,
+                             changelog_priv_t *priv,
+                             unsigned long ts, gf_boolean_t finale)
+{
+        int ret = -1;
+
+        ret = changelog_rollover_changelog (this, priv, ts);
+
+        if (!ret && !finale)
+                ret = changelog_open (this, priv);
+
+        return ret;
+}
+
+/**
+ * return the length of entry
+ */
+inline size_t
+changelog_entry_length ()
+{
+        return sizeof (changelog_log_data_t);
+}
+
+int
+changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last)
+{
+        struct timeval tv = {0,};
+
+        cld->cld_type = CHANGELOG_TYPE_ROLLOVER;
+
+        if (gettimeofday (&tv, NULL))
+                return -1;
+
+        cld->cld_roll_time = (unsigned long) tv.tv_sec;
+        cld->cld_finale = is_last;
+        return 0;
+}
+
+int
+changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len)
+{
+        return changelog_write (priv->changelog_fd, buffer, len);
+}
+
+inline int
+changelog_handle_change (xlator_t *this,
+                         changelog_priv_t *priv, changelog_log_data_t *cld)
+{
+        int ret = 0;
+
+        if (CHANGELOG_TYPE_IS_ROLLOVER (cld->cld_type)) {
+                changelog_encode_change(priv);
+                ret = changelog_start_next_change (this, priv,
+                                                   cld->cld_roll_time,
+                                                   cld->cld_finale);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Problem rolling over changelog(s)");
+                goto out;
+        }
+
+        /**
+         * case when there is reconfigure done (disabling changelog) and there
+         * are still fops that have updates in prgress.
+         */
+        if (priv->changelog_fd == -1)
+                return 0;
+
+        if (CHANGELOG_TYPE_IS_FSYNC (cld->cld_type)) {
+                ret = fsync (priv->changelog_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "fsync failed (reason: %s)",
+                                strerror (errno));
+                }
+                goto out;
+        }
+
+        ret = priv->ce->encode (this, cld);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error writing changelog to disk");
+        }
+
+ out:
+        return ret;
+}
+
+changelog_local_t *
+changelog_local_init (xlator_t *this, inode_t *inode,
+                      uuid_t gfid, int xtra_records,
+                      gf_boolean_t update_flag)
+{
+        changelog_local_t *local = NULL;
+        struct iobuf      *iobuf = NULL;
+
+        /**
+         * We relax the presence of inode if @update_flag is true.
+         * The caller (implmentation of the fop) needs to be careful to
+         * not blindly use local->inode.
+         */
+        if (!update_flag && !inode) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                                  "inode needed for version checking !!!");
+                goto out;
+        }
+
+        if (xtra_records) {
+                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
+                                    xtra_records * CHANGELOG_OPT_RECORD_LEN);
+                if (!iobuf)
+                        goto out;
+        }
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                CHANGELOG_IOBUF_UNREF (iobuf);
+                goto out;
+        }
+
+        local->update_no_check = update_flag;
+
+        uuid_copy (local->cld.cld_gfid, gfid);
+
+        local->cld.cld_iobuf = iobuf;
+        local->cld.cld_xtra_records = 0; /* set by the caller */
+
+        if (inode)
+                local->inode = inode_ref (inode);
+
+ out:
+        return local;
+}
+
+int
+changelog_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        changelog_inode_ctx_t *ctx = NULL;
+
+        inode_ctx_del (inode, this, &ctx_addr);
+        if (!ctx_addr)
+                return 0;
+
+        ctx = (changelog_inode_ctx_t *) (long) ctx_addr;
+        GF_FREE (ctx);
+
+        return 0;
+}
+
+int
+changelog_inject_single_event (xlator_t *this,
+                               changelog_priv_t *priv,
+                               changelog_log_data_t *cld)
+{
+        return priv->cd.dispatchfn (this, priv, priv->cd.cd_data, cld, NULL);
+}
+
+/**
+ * TODO: these threads have many thing in common (wake up after
+ * a certain time etc..). move them into separate routine.
+ */
+void *
+changelog_rollover (void *data)
+{
+        int                     ret   = 0;
+        xlator_t               *this  = NULL;
+        struct timeval          tv    = {0,};
+        changelog_log_data_t    cld   = {0,};
+        changelog_time_slice_t *slice = NULL;
+        changelog_priv_t       *priv  = data;
+
+        this = priv->cr.this;
+        slice = &priv->slice;
+
+        while (1) {
+                tv.tv_sec  = priv->rollover_time;
+                tv.tv_usec = 0;
+
+                ret = select (0, NULL, NULL, NULL, &tv);
+                if (ret)
+                        continue;
+
+                ret = changelog_fill_rollover_data (&cld, _gf_false);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to fill rollover data");
+                        continue;
+                }
+
+                LOCK (&priv->lock);
+                {
+                        ret = changelog_inject_single_event (this, priv, &cld);
+                        if (!ret)
+                                SLICE_VERSION_UPDATE (slice);
+                }
+                UNLOCK (&priv->lock);
+        }
+
+        return NULL;
+}
+
+void *
+changelog_fsync_thread (void *data)
+{
+        int                   ret  = 0;
+        xlator_t             *this = NULL;
+        struct timeval        tv   = {0,};
+        changelog_log_data_t  cld  = {0,};
+        changelog_priv_t     *priv = data;
+
+        this = priv->cf.this;
+        cld.cld_type = CHANGELOG_TYPE_FSYNC;
+
+        while (1) {
+                tv.tv_sec  = priv->fsync_interval;
+                tv.tv_usec = 0;
+
+                ret = select (0, NULL, NULL, NULL, &tv);
+                if (ret)
+                        continue;
+
+                ret = changelog_inject_single_event (this, priv, &cld);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to inject fsync event");
+        }
+
+        return NULL;
+}
+
+/* macros for inode/changelog version checks */
+
+#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type) do {       \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        LOCK (&priv->lock);                             \
+                        {                                               \
+                                *iver = slice->changelog_version[type]; \
+                        }                                               \
+                        UNLOCK (&priv->lock);                           \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd) do {    \
+                LOCK (&priv->lock);                                     \
+                {                                                       \
+                        upd = (ver == slice->changelog_version[type])   \
+                                ? _gf_false : _gf_true;                 \
+                }                                                       \
+                UNLOCK (&priv->lock);                                   \
+        } while (0)
+
+static int
+__changelog_inode_ctx_set (xlator_t *this,
+                           inode_t *inode, changelog_inode_ctx_t *ctx)
+{
+        uint64_t ctx_addr = (uint64_t) ctx;
+        return __inode_ctx_set (inode, this, &ctx_addr);
+}
+
+/**
+ * one shot routine to get the address and the value of a inode version
+ * for a particular type.
+ */
+static changelog_inode_ctx_t *
+__changelog_inode_ctx_get (xlator_t *this,
+                           inode_t *inode, unsigned long **iver,
+                           unsigned long *version, changelog_log_type type)
+{
+        int                    ret      = 0;
+        uint64_t               ctx_addr = 0;
+        changelog_inode_ctx_t *ctx      = NULL;
+
+        ret = __inode_ctx_get (inode, this, &ctx_addr);
+        if (ret < 0)
+                ctx_addr = 0;
+        if (ctx_addr != 0) {
+                ctx = (changelog_inode_ctx_t *) (long)ctx_addr;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_changelog_mt_inode_ctx_t);
+        if (!ctx)
+                goto out;
+
+        ret = __changelog_inode_ctx_set (this, inode, ctx);
+        if (ret) {
+                GF_FREE (ctx);
+                ctx = NULL;
+        }
+
+ out:
+        if (ctx && iver && version) {
+                *iver = CHANGELOG_INODE_VERSION_TYPE (ctx, type);
+                *version = **iver;
+        }
+
+        return ctx;
+}
+
+static changelog_inode_ctx_t *
+changelog_inode_ctx_get (xlator_t *this,
+                         inode_t *inode, unsigned long **iver,
+                         unsigned long *version, changelog_log_type type)
+{
+        changelog_inode_ctx_t *ctx = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __changelog_inode_ctx_get (this,
+                                                 inode, iver, version, type);
+        }
+        UNLOCK (&inode->lock);
+
+        return ctx;
+}
+
+/**
+ * This is the main update routine. Locking has been made granular so as to
+ * maximize parallelism of fops - I'll try to explain it below using execution
+ * timelines.
+ *
+ * Basically, the contention is between multiple execution threads of this
+ * routine and the roll-over thread. So, instead of having a big lock, we hold
+ * granular locks: inode->lock and priv->lock. Now I'll explain what happens
+ * when there is an update and a roll-over at just about the same time.
+ * NOTE:
+ *  - the dispatcher itself synchronizes updates via it's own lock
+ *  - the slice version in incremented by the roll-over thread
+ *
+ * Case 1: When the rollover thread wins before the inode version can be
+ * compared with the slice version.
+ *
+ *          [updater]                 |             [rollover]
+ *                                    |
+ *                                    |           <SLICE: 1, 1, 1>
+ * <changelog_update>                 |
+ *   <changelog_inode_ctx_get>        |
+ *      <CTX: 1, 1, 1>                |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 2, 2, 2>
+ *                                    |         UNLOCK (&priv->lock)
+ *                                    |
+ * LOCK (&priv->lock)                 |
+ *   <INODE_VERSION_EQUALS_SLICE>     |
+ *    I: 1 <-> S: 2                   |
+ *    update: true                    |
+ * UNLOCK (&priv->lock)               |
+ *                                    |
+ * <if update == true>                |
+ *  <dispath-update-event>            |
+ *  <INODE_VERSION_UPDATE>            |
+ *   LOCK (&inode->lock)              |
+ *    LOCK (&priv->lock)              |
+ *     <CTX: 2, 1, 1>                 |
+ *    UNLOCK (&priv->lock)            |
+ *   UNLOCK (&inode->lock)            |
+ *
+ * Therefore, the change gets recorded in the next change (no lost change). If
+ * the slice version was ahead of the inode version (say I:1, S: 2), then
+ * anyway the comparison would result in a update (I: 1, S: 3).
+ *
+ * If the rollover time is too less, then there is another contention when the
+ * updater tries to bring up inode version to the slice version (this is also
+ * the case when the roll-over thread wakes up during INODE_VERSION_UPDATE.
+ *
+ *   <CTX: 1, 1, 1>                   |       <SLICE: 2, 2, 2>
+ *                                    |
+ *                                    |
+ * <dispath-update-event>             |
+ * <INODE_VERSION_UPDATE>             |
+ *  LOCK (&inode->lock)               |
+ *   LOCK (&priv->lock)               |
+ *    <CTX: 2, 1, 1>                  |
+ *   UNLOCK (&priv->lock)             |
+ *  UNLOCK (&inode->lock)             |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 3, 3, 3>
+ *                                    |         UNLOCK (&priv->lock)
+ *
+ *
+ * Case 2: When the fop thread wins
+ *
+ *          [updater]                 |             [rollover]
+ *                                    |
+ *                                    |           <SLICE: 1, 1, 1>
+ * <changelog_update>                 |
+ *   <changelog_inode_ctx_get>        |
+ *      <CTX: 0, 0, 0>                |
+ *                                    |
+ * LOCK (&priv->lock)                 |
+ *   <INODE_VERSION_EQUALS_SLICE>     |
+ *    I: 0 <-> S: 1                   |
+ *    update: true                    |
+ * UNLOCK (&priv->lock)               |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 2, 2, 2>
+ *                                    |         UNLOCK (&priv->lock)
+ * <if update == true>                |
+ *  <dispath-update-event>            |
+ *  <INODE_VERSION_UPDATE>            |
+ *   LOCK (&inode->lock)              |
+ *    LOCK (&priv->lock)              |
+ *     <CTX: 2, 0, 0>                 |
+ *    UNLOCK (&priv->lock)            |
+ *   UNLOCK (&inode->lock)            |
+ *
+ * Here again, if the inode version was equal to the slice version (I: 1, S: 1)
+ * then there is no need to record an update (as the equality of the two version
+ * signifies an update was recorded in the current time slice).
+ */
+inline void
+changelog_update (xlator_t *this, changelog_priv_t *priv,
+                  changelog_local_t *local, changelog_log_type type)
+{
+        int                     ret        = 0;
+        unsigned long          *iver       = NULL;
+        unsigned long           version    = 0;
+        inode_t                *inode      = NULL;
+        changelog_time_slice_t *slice      = NULL;
+        changelog_inode_ctx_t  *ctx        = NULL;
+        changelog_log_data_t   *cld_0      = NULL;
+        changelog_log_data_t   *cld_1      = NULL;
+        changelog_local_t      *next_local = NULL;
+        gf_boolean_t            need_upd   = _gf_true;
+
+        slice = &priv->slice;
+
+        /**
+         * for fops that do not require inode version checking
+         */
+        if (local->update_no_check)
+                goto update;
+
+        inode = local->inode;
+
+        ctx = changelog_inode_ctx_get (this,
+                                       inode, &iver, &version, type);
+        if (!ctx)
+                goto update;
+
+        INODE_VERSION_EQUALS_SLICE (priv, version, slice, type, need_upd);
+
+ update:
+        if (need_upd) {
+                cld_0 = &local->cld;
+                cld_0->cld_type = type;
+
+                if ( (next_local = local->prev_entry) != NULL ) {
+                        cld_1 = &next_local->cld;
+                        cld_1->cld_type = type;
+                }
+
+                ret = priv->cd.dispatchfn (this, priv,
+                                           priv->cd.cd_data, cld_0, cld_1);
+
+                /**
+                 * update after the dispatcher has successfully done
+                 * it's job.
+                 */
+                if (!local->update_no_check && iver && !ret)
+                        INODE_VERSION_UPDATE (priv, inode, iver, slice, type);
+        }
+
+        return;
+}
diff --git a/xlators/features/changelog/src/changelog-helpers.h b/xlators/features/changelog/src/changelog-helpers.h
new file mode 100644
index 000000000..ad79636b0
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-helpers.h
@@ -0,0 +1,395 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_HELPERS_H
+#define _CHANGELOG_HELPERS_H
+
+#include "locking.h"
+#include "timer.h"
+#include "pthread.h"
+#include "iobuf.h"
+
+#include "changelog-misc.h"
+
+/**
+ * the changelog entry
+ */
+typedef struct changelog_log_data {
+        /* rollover related */
+        unsigned long cld_roll_time;
+
+        /* reopen changelog? */
+        gf_boolean_t cld_finale;
+
+        changelog_log_type cld_type;
+
+        /**
+         * sincd gfid is _always_ a necessity, it's not a part
+         * of the iobuf. by doing this we do not add any overhead
+         * for data and metadata related fops.
+         */
+        uuid_t        cld_gfid;
+
+        /**
+         * iobufs are used for optionals records: pargfid, path,
+         * write offsets etc.. It's the fop implementers job
+         * to allocate (iobuf_get() in the fop) and get unref'ed
+         * in the callback (CHANGELOG_STACK_UNWIND).
+         */
+        struct iobuf *cld_iobuf;
+
+#define cld_ptr cld_iobuf->ptr
+
+        /**
+         * after allocation you can point this to the length of
+         * usable data, but make sure it does not exceed the
+         * the size of the requested iobuf.
+         */
+        size_t        cld_iobuf_len;
+
+#define cld_ptr_len cld_iobuf_len
+
+        /**
+         * number of optional records
+         */
+        int cld_xtra_records;
+} changelog_log_data_t;
+
+/**
+ * holder for dispatch function and private data
+ */
+
+typedef struct changelog_priv changelog_priv_t;
+
+typedef struct changelog_dispatcher {
+        void *cd_data;
+        int (*dispatchfn) (xlator_t *, changelog_priv_t *, void *,
+                           changelog_log_data_t *, changelog_log_data_t *);
+} changelog_dispatcher_t;
+
+struct changelog_bootstrap {
+        changelog_mode_t mode;
+        int (*ctor) (xlator_t *, changelog_dispatcher_t *);
+        int (*dtor) (xlator_t *, changelog_dispatcher_t *);
+};
+
+struct changelog_encoder {
+        changelog_encoder_t encoder;
+        int (*encode) (xlator_t *, changelog_log_data_t *);
+};
+
+
+/* xlator private */
+
+typedef struct changelog_time_slice {
+        /**
+         * just in case we need nanosecond granularity some day.
+         * field is unused as of now (maybe we'd need it later).
+         */
+        struct timeval tv_start;
+
+        /**
+         * version of changelog file, incremented each time changes
+         * rollover.
+         */
+        unsigned long changelog_version[CHANGELOG_MAX_TYPE];
+} changelog_time_slice_t;
+
+typedef struct changelog_rollover {
+        /* rollover thread */
+        pthread_t rollover_th;
+
+        xlator_t *this;
+} changelog_rollover_t;
+
+typedef struct changelog_fsync {
+        /* fsync() thread */
+        pthread_t fsync_th;
+
+        xlator_t *this;
+} changelog_fsync_t;
+
+# define CHANGELOG_MAX_CLIENTS  5
+typedef struct changelog_notify {
+        /* reader end of the pipe */
+        int rfd;
+
+        /* notifier thread */
+        pthread_t notify_th;
+
+        /* unique socket path */
+        char sockpath[PATH_MAX];
+
+        int socket_fd;
+
+        /**
+         * simple array of accept()'ed fds. Not scalable at all
+         * for large number of clients, but it's okay as we have
+         * a ahrd limit in this version (@CHANGELOG_MAX_CLIENTS).
+         */
+        int client_fd[CHANGELOG_MAX_CLIENTS];
+
+        xlator_t *this;
+} changelog_notify_t;
+
+struct changelog_priv {
+        gf_boolean_t active;
+
+        /* to generate unique socket file per brick */
+        char *changelog_brick;
+
+        /* logging directory */
+        char *changelog_dir;
+
+        /* one file for all changelog types */
+        int changelog_fd;
+
+        gf_lock_t lock;
+
+        /* writen end of the pipe */
+        int wfd;
+
+        /* rollover time */
+        int32_t rollover_time;
+
+        /* fsync() interval */
+        int32_t fsync_interval;
+
+        /* changelog type maps */
+        const char *maps[CHANGELOG_MAX_TYPE];
+
+        /* time slicer */
+        changelog_time_slice_t slice;
+
+        /* context of the updater */
+        changelog_dispatcher_t cd;
+
+        /* context of the rollover thread */
+        changelog_rollover_t cr;
+
+        /* context of fsync thread */
+        changelog_fsync_t cf;
+
+        /* context of the notifier thread */
+        changelog_notify_t cn;
+
+        /* operation mode */
+        changelog_mode_t op_mode;
+
+        /* bootstrap routine for 'current' logger */
+        struct changelog_bootstrap *cb;
+
+        /* encoder mode */
+        changelog_encoder_t encode_mode;
+
+        /* encoder */
+        struct changelog_encoder *ce;
+};
+
+struct changelog_local {
+        inode_t              *inode;
+        gf_boolean_t          update_no_check;
+
+        changelog_log_data_t  cld;
+
+        /**
+         * ->prev_entry is used in cases when there needs to be
+         * additional changelog entry for the parent (eg. rename)
+         * It's analogous to ->next in single linked list world,
+         * but we call it as ->prev_entry... ha ha ha
+         */
+        struct changelog_local *prev_entry;
+};
+
+typedef struct changelog_local changelog_local_t;
+
+/* inode version is stored in inode ctx */
+typedef struct changelog_inode_ctx {
+        unsigned long iversion[CHANGELOG_MAX_TYPE];
+} changelog_inode_ctx_t;
+
+#define CHANGELOG_INODE_VERSION_TYPE(ctx, type)  &(ctx->iversion[type])
+
+/**
+ * Optional Records:
+ *  fops that need to save additional information request a array of
+ *  @changelog_opt_t struct. The array is allocated via @iobufs.
+ */
+typedef enum {
+        CHANGELOG_OPT_REC_FOP,
+        CHANGELOG_OPT_REC_ENTRY,
+} changelog_optional_rec_type_t;
+
+struct changelog_entry_fields {
+        uuid_t  cef_uuid;
+        char   *cef_bname;
+};
+
+typedef struct {
+        /**
+         * @co_covert can be used to do post-processing of the record before
+         * it's persisted to the CHANGELOG. If this is NULL, then the record
+         * is persisted as per it's in memory format.
+         */
+        size_t (*co_convert) (void *data, char *buffer, gf_boolean_t encode);
+
+        /* release routines */
+        void (*co_free) (void *data);
+
+        /* type of the field */
+        changelog_optional_rec_type_t co_type;
+
+        /**
+         * sizeof of the 'valid' field in the union. This field is not used if
+         * @co_convert is specified.
+         */
+        size_t co_len;
+
+        union {
+                glusterfs_fop_t co_fop;
+                struct changelog_entry_fields co_entry;
+        };
+} changelog_opt_t;
+
+#define CHANGELOG_OPT_RECORD_LEN  sizeof (changelog_opt_t)
+
+/**
+ * helpers routines
+ */
+
+void
+changelog_thread_cleanup (xlator_t *this, pthread_t thr_id);
+inline void *
+changelog_get_usable_buffer (changelog_local_t *local);
+inline void
+changelog_set_usable_record_and_length (changelog_local_t *local,
+                                        size_t len, int xr);
+void
+changelog_local_cleanup (xlator_t *xl, changelog_local_t *local);
+changelog_local_t *
+changelog_local_init (xlator_t *this, inode_t *inode, uuid_t gfid,
+                      int xtra_records, gf_boolean_t update_flag);
+int
+changelog_start_next_change (xlator_t *this,
+                             changelog_priv_t *priv,
+                             unsigned long ts, gf_boolean_t finale);
+int
+changelog_open (xlator_t *this, changelog_priv_t *priv);
+int
+changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last);
+int
+changelog_inject_single_event (xlator_t *this,
+                               changelog_priv_t *priv,
+                               changelog_log_data_t *cld);
+inline size_t
+changelog_entry_length ();
+inline int
+changelog_write (int fd, char *buffer, size_t len);
+int
+changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len);
+inline int
+changelog_handle_change (xlator_t *this,
+                         changelog_priv_t *priv, changelog_log_data_t *cld);
+inline void
+changelog_update (xlator_t *this, changelog_priv_t *priv,
+                  changelog_local_t *local, changelog_log_type type);
+void *
+changelog_rollover (void *data);
+void *
+changelog_fsync_thread (void *data);
+int
+changelog_forget (xlator_t *this, inode_t *inode);
+
+/* macros */
+
+#define CHANGELOG_STACK_UNWIND(fop, frame, params ...) do {             \
+                changelog_local_t *__local = NULL;                      \
+                xlator_t          *__xl    = NULL;                      \
+                if (frame) {                                            \
+                        __local      = frame->local;                    \
+                        __xl         = frame->this;                     \
+                        frame->local = NULL;                            \
+                }                                                       \
+                STACK_UNWIND_STRICT (fop, frame, params);               \
+                changelog_local_cleanup (__xl, __local);                \
+                if (__local && __local->prev_entry)                     \
+                        changelog_local_cleanup (__xl,                  \
+                                                 __local->prev_entry);  \
+        } while (0)
+
+#define CHANGELOG_IOBUF_REF(iobuf) do {         \
+                if (iobuf)                      \
+                        iobuf_ref (iobuf);      \
+        } while (0)
+
+#define CHANGELOG_IOBUF_UNREF(iobuf) do {       \
+                if (iobuf)                      \
+                        iobuf_unref (iobuf);    \
+        } while (0)
+
+#define CHANGELOG_FILL_BUFFER(buffer, off, val, len) do {       \
+                memcpy (buffer + off, val, len);                \
+                off += len;                                     \
+        } while (0)
+
+#define SLICE_VERSION_UPDATE(slice) do {                \
+                int i = 0;                              \
+                for (; i < CHANGELOG_MAX_TYPE; i++) {   \
+                        slice->changelog_version[i]++;  \
+                }                                       \
+        } while (0)
+
+#define CHANGLOG_FILL_FOP_NUMBER(co, fop, converter, xlen) do { \
+                co->co_convert = converter;                     \
+                co->co_free = NULL;                             \
+                co->co_type = CHANGELOG_OPT_REC_FOP;            \
+                co->co_fop = fop;                               \
+                xlen += sizeof (fop);                           \
+        } while (0)
+
+#define CHANGELOG_FILL_ENTRY(co, pargfid, bname,                        \
+                             converter, freefn, xlen, label)            \
+        do {                                                            \
+                co->co_convert = converter;                             \
+                co->co_free = freefn;                                   \
+                co->co_type = CHANGELOG_OPT_REC_ENTRY;                  \
+                uuid_copy (co->co_entry.cef_uuid, pargfid);             \
+                co->co_entry.cef_bname = gf_strdup(bname);              \
+                if (!co->co_entry.cef_bname)                            \
+                        goto label;                                     \
+                xlen += (UUID_CANONICAL_FORM_LEN + strlen (bname));     \
+        } while (0)
+
+#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                  \
+        local = changelog_local_init (this, inode, gfid, xrec, _gf_false)
+
+#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)          \
+        local = changelog_local_init (this, inode, gfid, xrec, _gf_true)
+
+#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label) do { \
+                if (!priv->active)                              \
+                        goto label;                             \
+                /* ignore rebalance process's activity. */      \
+                if (frame->root->pid == GF_CLIENT_PID_DEFRAG)   \
+                        goto label;                             \
+        } while (0)
+
+/* ignore internal fops */
+#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(dict, label) do {           \
+                if (dict && dict_get (dict, GLUSTERFS_INTERNAL_FOP_KEY)) \
+                        goto label;                                     \
+        } while (0)
+
+#define CHANGELOG_COND_GOTO(priv, cond, label) do {                    \
+                if (!priv->active || cond)                             \
+                        goto label;                                    \
+        } while (0)
+
+#endif /* _CHANGELOG_HELPERS_H */
diff --git a/xlators/features/changelog/src/changelog-mem-types.h b/xlators/features/changelog/src/changelog-mem-types.h
new file mode 100644
index 000000000..d72464eab
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-mem-types.h
@@ -0,0 +1,29 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_MEM_TYPES_H
+#define _CHANGELOG_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_changelog_mem_types {
+        gf_changelog_mt_priv_t                  = gf_common_mt_end + 1,
+        gf_changelog_mt_str_t                   = gf_common_mt_end + 2,
+        gf_changelog_mt_batch_t                 = gf_common_mt_end + 3,
+        gf_changelog_mt_rt_t                    = gf_common_mt_end + 4,
+        gf_changelog_mt_inode_ctx_t             = gf_common_mt_end + 5,
+        gf_changelog_mt_libgfchangelog_t        = gf_common_mt_end + 6,
+        gf_changelog_mt_libgfchangelog_rl_t     = gf_common_mt_end + 7,
+        gf_changelog_mt_libgfchangelog_dirent_t = gf_common_mt_end + 8,
+        gf_changelog_mt_changelog_buffer_t      = gf_common_mt_end + 9,
+        gf_changelog_mt_end
+};
+
+#endif
diff --git a/xlators/features/changelog/src/changelog-misc.h b/xlators/features/changelog/src/changelog-misc.h
new file mode 100644
index 000000000..0712a3771
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-misc.h
@@ -0,0 +1,101 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_MISC_H
+#define _CHANGELOG_MISC_H
+
+#include "glusterfs.h"
+#include "common-utils.h"
+
+#define CHANGELOG_MAX_TYPE  3
+#define CHANGELOG_FILE_NAME "CHANGELOG"
+
+#define CHANGELOG_VERSION_MAJOR  1
+#define CHANGELOG_VERSION_MINOR  0
+
+#define CHANGELOG_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/changelog-%s.sock"
+
+/**
+ * header starts with the version and the format of the changelog.
+ * 'version' not much of a use now.
+ */
+#define CHANGELOG_HEADER                                                \
+        "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
+
+#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len) do {      \
+                char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};             \
+                md5_wrapper((unsigned char *) brick_path,               \
+                            strlen(brick_path),                         \
+                            md5_sum);                                   \
+                (void) snprintf (sockpath, len,                         \
+                                 CHANGELOG_UNIX_SOCK, md5_sum);         \
+        } while (0)
+
+/**
+ * ... used by libgfchangelog.
+ */
+#define CHANGELOG_GET_ENCODING(fd, buffer, len, enc, enc_len) do {      \
+                FILE *fp;                                               \
+                int fd_dup, maj, min;                                   \
+                                                                        \
+                enc = -1;                                               \
+                fd_dup = dup (fd);                                      \
+                                                                        \
+                if (fd_dup != -1) {                                     \
+                        fp = fdopen (fd_dup, "r");                      \
+                        if (fp) {                                       \
+                                if (fgets (buffer, len, fp)) {          \
+                                        elen = strlen (buffer);         \
+                                        sscanf (buffer,                 \
+                                                CHANGELOG_HEADER,       \
+                                                &maj, &min, &enc);      \
+                                }                                       \
+                                fclose (fp);                            \
+                        } else {                                        \
+                                close (fd_dup);                         \
+                        }                                               \
+                }                                                       \
+        } while (0)
+
+/**
+ * everything after 'CHANGELOG_TYPE_ENTRY' are internal types
+ * (ie. none of the fops trigger this type of event), hence
+ * CHANGELOG_MAX_TYPE = 3
+ */
+typedef enum {
+        CHANGELOG_TYPE_DATA = 0,
+        CHANGELOG_TYPE_METADATA,
+        CHANGELOG_TYPE_ENTRY,
+        CHANGELOG_TYPE_ROLLOVER,
+        CHANGELOG_TYPE_FSYNC,
+} changelog_log_type;
+
+/* operation modes - RT for now */
+typedef enum {
+        CHANGELOG_MODE_RT = 0,
+} changelog_mode_t;
+
+/* encoder types */
+
+typedef enum {
+        CHANGELOG_ENCODE_MIN = 0,
+        CHANGELOG_ENCODE_BINARY,
+        CHANGELOG_ENCODE_ASCII,
+        CHANGELOG_ENCODE_MAX,
+} changelog_encoder_t;
+
+#define CHANGELOG_VALID_ENCODING(enc)                                   \
+        (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
+
+#define CHANGELOG_TYPE_IS_ENTRY(type)  (type == CHANGELOG_TYPE_ENTRY)
+#define CHANGELOG_TYPE_IS_ROLLOVER(type)  (type == CHANGELOG_TYPE_ROLLOVER)
+#define CHANGELOG_TYPE_IS_FSYNC(type)  (type == CHANGELOG_TYPE_FSYNC)
+
+#endif /* _CHANGELOG_MISC_H */
diff --git a/xlators/features/changelog/src/changelog-notifier.c b/xlators/features/changelog/src/changelog-notifier.c
new file mode 100644
index 000000000..1f8b31253
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-notifier.c
@@ -0,0 +1,314 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-notifier.h"
+
+#include <pthread.h>
+
+inline static void
+changelog_notify_clear_fd (changelog_notify_t *cn, int i)
+{
+        cn->client_fd[i] = -1;
+}
+
+inline static void
+changelog_notify_save_fd (changelog_notify_t *cn, int i, int fd)
+{
+        cn->client_fd[i] = fd;
+}
+
+static int
+changelog_notify_insert_fd (xlator_t *this, changelog_notify_t *cn, int fd)
+{
+        int i   = 0;
+        int ret = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        break;
+        }
+
+        if (i == CHANGELOG_MAX_CLIENTS) {
+                /**
+                 * this case should not be hit as listen() would limit
+                 * the number of completely established connections.
+                 */
+                gf_log (this->name, GF_LOG_WARNING,
+                        "hit max client limit (%d)", CHANGELOG_MAX_CLIENTS);
+                ret = -1;
+        }
+        else
+                changelog_notify_save_fd (cn, i, fd);
+
+        return ret;
+}
+
+static void
+changelog_notify_fill_rset (changelog_notify_t *cn, fd_set *rset, int *maxfd)
+{
+        int i = 0;
+
+        FD_ZERO (rset);
+
+        FD_SET (cn->socket_fd, rset);
+        *maxfd = cn->socket_fd;
+
+        FD_SET (cn->rfd, rset);
+        *maxfd = max (*maxfd, cn->rfd);
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] != -1) {
+                        FD_SET (cn->client_fd[i], rset);
+                        *maxfd = max (*maxfd, cn->client_fd[i]);
+                }
+        }
+
+        *maxfd = *maxfd + 1;
+}
+
+static int
+changelog_notify_client (changelog_notify_t *cn, char *path, ssize_t len)
+{
+        int i = 0;
+        int ret = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        continue;
+
+                if (changelog_write (cn->client_fd[i],
+                                     path, len)) {
+                        ret = -1;
+
+                        close (cn->client_fd[i]);
+                        changelog_notify_clear_fd (cn, i);
+                }
+        }
+
+        return ret;
+}
+
+static void
+changelog_notifier_init (changelog_notify_t *cn)
+{
+        int i = 0;
+
+        cn->socket_fd = -1;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                changelog_notify_clear_fd (cn, i);
+        }
+}
+
+static void
+changelog_close_client_conn (changelog_notify_t *cn)
+{
+        int i = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        continue;
+
+                close (cn->client_fd[i]);
+                changelog_notify_clear_fd (cn, i);
+        }
+}
+
+static void
+changelog_notifier_cleanup (void *arg)
+{
+        changelog_notify_t *cn = NULL;
+
+        cn = (changelog_notify_t *) arg;
+
+        changelog_close_client_conn (cn);
+
+        if (cn->socket_fd != -1)
+                close (cn->socket_fd);
+
+        if (cn->rfd)
+                close (cn->rfd);
+
+        if (unlink (cn->sockpath))
+                gf_log ("", GF_LOG_WARNING,
+                        "could not unlink changelog socket file"
+                        " %s (reason: %s", cn->sockpath, strerror (errno));
+}
+
+void *
+changelog_notifier (void *data)
+{
+        int                 i         = 0;
+        int                 fd        = 0;
+        int                 max_fd    = 0;
+        int                 len       = 0;
+        ssize_t             readlen   = 0;
+        xlator_t           *this      = NULL;
+        changelog_priv_t   *priv      = NULL;
+        changelog_notify_t *cn        = NULL;
+        struct sockaddr_un  local     = {0,};
+        char path[PATH_MAX]           = {0,};
+        char abspath[PATH_MAX]        = {0,};
+
+        char buffer;
+        fd_set rset;
+
+        priv = (changelog_priv_t *) data;
+
+        cn = &priv->cn;
+        this = cn->this;
+
+        pthread_cleanup_push (changelog_notifier_cleanup, cn);
+
+        changelog_notifier_init (cn);
+
+        cn->socket_fd = socket (AF_UNIX, SOCK_STREAM, 0);
+        if (cn->socket_fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "changelog socket error (reason: %s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
+                                    cn->sockpath, PATH_MAX);
+        if (unlink (cn->sockpath) < 0) {
+                if (errno != ENOENT) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not unlink changelog socket file (%s)"
+                                " (reason: %s)",
+                                CHANGELOG_UNIX_SOCK, strerror (errno));
+                        goto cleanup;
+                }
+        }
+
+        local.sun_family = AF_UNIX;
+        strcpy (local.sun_path, cn->sockpath);
+
+        len = strlen (local.sun_path) + sizeof (local.sun_family);
+
+        /* bind to the unix domain socket */
+        if (bind (cn->socket_fd, (struct sockaddr *) &local, len) < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not bind to changelog socket (reason: %s)",
+                        strerror (errno));
+                goto cleanup;
+        }
+
+        /* listen for incoming connections */
+        if (listen (cn->socket_fd, CHANGELOG_MAX_CLIENTS) < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "listen() error on changelog socket (reason: %s)",
+                        strerror (errno));
+                goto cleanup;
+        }
+
+        /**
+         * simple select() on all to-be-read file descriptors. This method
+         * though old school works pretty well when you have a handfull of
+         * fd's to be watched (clients).
+         *
+         * Future TODO: move this to epoll based notification facility if
+         *              number of clients increase.
+         */
+        for (;;) {
+                changelog_notify_fill_rset (cn, &rset, &max_fd);
+
+                if (select (max_fd, &rset, NULL, NULL, NULL) < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "select() returned -1 (reason: %s)",
+                                strerror (errno));
+                        sleep (2);
+                        continue;
+                }
+
+                if (FD_ISSET (cn->socket_fd, &rset)) {
+                        fd = accept (cn->socket_fd, NULL, NULL);
+                        if (fd < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "accept error on changelog socket"
+                                        " (reason: %s)", strerror (errno));
+                        } else if (changelog_notify_insert_fd (this, cn, fd)) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "hit max client limit");
+                        }
+                }
+
+                if (FD_ISSET (cn->rfd, &rset)) {
+                        /**
+                         * read changelog filename and notify all connected
+                         * clients.
+                         */
+                        readlen = 0;
+                        while (readlen < PATH_MAX) {
+                                len = read (cn->rfd, &path[readlen++], 1);
+                                if (len == -1) {
+                                        break;
+                                }
+
+                                if (len == 0) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "rollover thread sent EOF"
+                                                " on pipe - possibly a crash.");
+                                        /* be blunt and close all connections */
+                                        pthread_exit(NULL);
+                                }
+
+                                if (path[readlen - 1] == '\0')
+                                        break;
+                        }
+
+                        /* should we close all client connections here too? */
+                        if (len < 0 || readlen == PATH_MAX) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not get pathname from rollover"
+                                        " thread or pathname too long");
+                                goto process_rest;
+                        }
+
+                        (void) snprintf (abspath, PATH_MAX,
+                                         "%s/%s", priv->changelog_dir, path);
+                        if (changelog_notify_client (cn, abspath,
+                                                     strlen (abspath) + 1))
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not notify some clients with new"
+                                        " changelogs");
+                }
+
+        process_rest:
+                for (i = 0; i < CHANGELOG_MAX_CLIENTS; i++) {
+                        if ( (fd = cn->client_fd[i]) == -1 )
+                                continue;
+
+                        if (FD_ISSET (fd, &rset)) {
+                                /**
+                                 * the only data we accept from the client is a
+                                 * disconnect. Anything else is treated as bogus
+                                 * and is silently discarded (also warned!!!).
+                                 */
+                                if ( (readlen = read (fd, &buffer, 1)) <= 0 ) {
+                                        close (fd);
+                                        changelog_notify_clear_fd (cn, i);
+                                } else {
+                                        /* silently discard data and log */
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "misbehaving changelog client");
+                                }
+                        }
+                }
+
+        }
+
+ cleanup:;
+        pthread_cleanup_pop (1);
+
+ out:
+        return NULL;
+}
diff --git a/xlators/features/changelog/src/changelog-notifier.h b/xlators/features/changelog/src/changelog-notifier.h
new file mode 100644
index 000000000..55e728356
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-notifier.h
@@ -0,0 +1,19 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_NOTIFIER_H
+#define _CHANGELOG_NOTIFIER_H
+
+#include "changelog-helpers.h"
+
+void *
+changelog_notifier (void *data);
+
+#endif
diff --git a/xlators/features/changelog/src/changelog-rt.c b/xlators/features/changelog/src/changelog-rt.c
new file mode 100644
index 000000000..c147f68ca
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-rt.c
@@ -0,0 +1,72 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "changelog-rt.h"
+#include "changelog-mem-types.h"
+
+int
+changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd)
+{
+        changelog_rt_t *crt = NULL;
+
+        crt = GF_CALLOC (1, sizeof (*crt),
+                         gf_changelog_mt_rt_t);
+        if (!crt)
+                return -1;
+
+        LOCK_INIT (&crt->lock);
+
+        cd->cd_data = crt;
+        cd->dispatchfn = &changelog_rt_enqueue;
+
+        return 0;
+}
+
+int
+changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd)
+{
+        changelog_rt_t *crt = NULL;
+
+        crt = cd->cd_data;
+
+        LOCK_DESTROY (&crt->lock);
+        GF_FREE (crt);
+
+        return 0;
+}
+
+int
+changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1)
+{
+        int             ret = 0;
+        changelog_rt_t *crt = NULL;
+
+        crt = (changelog_rt_t *) cbatch;
+
+        LOCK (&crt->lock);
+        {
+                ret = changelog_handle_change (this, priv, cld_0);
+                if (!ret && cld_1)
+                        ret = changelog_handle_change (this, priv, cld_1);
+        }
+        UNLOCK (&crt->lock);
+
+        return ret;
+}
diff --git a/xlators/features/changelog/src/changelog-rt.h b/xlators/features/changelog/src/changelog-rt.h
new file mode 100644
index 000000000..1fc2bbc5b
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-rt.h
@@ -0,0 +1,33 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_RT_H
+#define _CHANGELOG_RT_H
+
+#include "locking.h"
+#include "timer.h"
+#include "pthread.h"
+
+#include "changelog-helpers.h"
+
+/* unused as of now - may be you would need it later */
+typedef struct changelog_rt {
+        gf_lock_t lock;
+} changelog_rt_t;
+
+int
+changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd);
+int
+changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd);
+int
+changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1);
+
+#endif /* _CHANGELOG_RT_H */
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
new file mode 100644
index 000000000..cea0e8c70
--- /dev/null
+++ b/xlators/features/changelog/src/changelog.c
@@ -0,0 +1,1477 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+#include "iobuf.h"
+
+#include "changelog-rt.h"
+
+#include "changelog-encoders.h"
+#include "changelog-mem-types.h"
+
+#include <pthread.h>
+
+#include "changelog-notifier.h"
+
+static struct changelog_bootstrap
+cb_bootstrap[] = {
+        {
+                .mode = CHANGELOG_MODE_RT,
+                .ctor = changelog_rt_init,
+                .dtor = changelog_rt_fini,
+        },
+};
+
+/* Entry operations - TYPE III */
+
+/**
+ * entry operations do not undergo inode version checking.
+ */
+
+/* {{{ */
+
+/* rmdir */
+
+int32_t
+changelog_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                                preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_rmdir (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, int xflags, dict_t *xdata)
+{
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local,
+                                NULL, loc->inode->gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_rmdir_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
+                    loc, xflags, xdata);
+        return 0;
+}
+
+/* unlink */
+
+int32_t
+changelog_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                                preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_unlink (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, int xflags, dict_t *xdata)
+{
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (xdata, wind);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, loc->inode->gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_unlink_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
+                    loc, xflags, xdata);
+        return 0;
+}
+
+/* rename */
+
+int32_t
+changelog_rename_cbk (call_frame_t *frame,
+                      void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *buf, struct iatt *preoldparent,
+                      struct iatt *postoldparent, struct iatt *prenewparent,
+                      struct iatt *postnewparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (rename, frame, op_ret, op_errno,
+                                buf, preoldparent, postoldparent,
+                                prenewparent, postnewparent, xdata);
+        return 0;
+}
+
+
+int32_t
+changelog_rename (call_frame_t *frame, xlator_t *this,
+                  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        size_t            xtra_len  = 0;
+        changelog_priv_t *priv      = NULL;
+        changelog_opt_t  *co        = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        /* 3 == fop + oldloc + newloc */
+        CHANGELOG_INIT_NOCHECK (this, frame->local,
+                                NULL, oldloc->inode->gfid, 3);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, oldloc->pargfid, oldloc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 3);
+
+ wind:
+        STACK_WIND (frame, changelog_rename_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+/* link */
+
+int32_t
+changelog_link_cbk (call_frame_t *frame,
+                    void *cookie, xlator_t *this, int32_t op_ret,
+                    int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (link, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_link (call_frame_t *frame,
+                xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (xdata, wind);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, oldloc->gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_link_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+/* mkdir */
+
+int32_t
+changelog_mkdir_cbk (call_frame_t *frame,
+                     void *cookie, xlator_t *this, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_mkdir (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+{
+        int               ret      = -1;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_mkdir_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+        return 0;
+}
+
+/* symlink */
+
+int32_t
+changelog_symlink_cbk (call_frame_t *frame,
+                       void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (symlink, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_symlink (call_frame_t *frame, xlator_t *this,
+                   const char *linkname, loc_t *loc,
+                   mode_t umask, dict_t *xdata)
+{
+        int               ret      = -1;
+        size_t            xtra_len = 0;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_symlink_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
+                    linkname, loc, umask, xdata);
+        return 0;
+}
+
+/* mknod */
+
+int32_t
+changelog_mknod_cbk (call_frame_t *frame,
+                     void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (mknod, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_mknod (call_frame_t *frame,
+                 xlator_t *this, loc_t *loc,
+                 mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+{
+        int               ret      = -1;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        size_t            xtra_len = 0;
+        changelog_priv_t *priv     = NULL;
+        changelog_opt_t  *co       = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_mknod_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
+                    loc, mode, dev, umask, xdata);
+        return 0;
+}
+
+/* creat */
+
+int32_t
+changelog_create_cbk (call_frame_t *frame,
+                      void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      fd_t *fd, inode_t *inode, struct iatt *buf,
+                      struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (create, frame,
+                                op_ret, op_errno, fd, inode,
+                                buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_create (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, int32_t flags, mode_t mode,
+                  mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int               ret      = -1;
+        uuid_t            gfid     = {0,};
+        void             *uuid_req = NULL;
+        changelog_opt_t  *co       = NULL;
+        changelog_priv_t *priv     = NULL;
+        size_t            xtra_len = 0;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto wind;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        /* init with two extra records */
+        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
+        if (!frame->local)
+                goto wind;
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto wind;
+
+        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
+
+ wind:
+        STACK_WIND (frame, changelog_create_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+}
+
+/* }}} */
+
+
+/* Metadata modification fops - TYPE II */
+
+/* {{{ */
+
+/* {f}setattr */
+
+int32_t
+changelog_fsetattr_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this, int32_t op_ret,
+                        int32_t op_errno, struct iatt *preop_stbuf,
+                        struct iatt *postop_stbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
+                                preop_stbuf, postop_stbuf, xdata);
+
+        return 0;
+
+
+}
+
+int32_t
+changelog_fsetattr (call_frame_t *frame,
+                    xlator_t *this, fd_t *fd,
+                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_fsetattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+        return 0;
+
+
+}
+
+int32_t
+changelog_setattr_cbk (call_frame_t *frame,
+                       void *cookie, xlator_t *this, int32_t op_ret,
+                       int32_t op_errno, struct iatt *preop_stbuf,
+                       struct iatt *postop_stbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (setattr, frame, op_ret, op_errno,
+                                preop_stbuf, postop_stbuf, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_setattr (call_frame_t *frame,
+                   xlator_t *this, loc_t *loc,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_setattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+        return 0;
+}
+
+/* {f}removexattr */
+
+int32_t
+changelog_fremovexattr_cbk (call_frame_t *frame,
+                            void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_fremovexattr (call_frame_t *frame, xlator_t *this,
+                        fd_t *fd, const char *name, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_fremovexattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+int32_t
+changelog_removexattr_cbk (call_frame_t *frame,
+                           void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_removexattr (call_frame_t *frame, xlator_t *this,
+                       loc_t *loc, const char *name, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_removexattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+/* {f}setxattr */
+
+int32_t
+changelog_setxattr_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_setxattr (call_frame_t *frame,
+                    xlator_t *this, loc_t *loc,
+                    dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_setxattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+changelog_fsetxattr_cbk (call_frame_t *frame,
+                         void *cookie, xlator_t *this, int32_t op_ret,
+                         int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_fsetxattr (call_frame_t *frame,
+                     xlator_t *this, fd_t *fd, dict_t *dict,
+                     int32_t flags, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_fsetxattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
+}
+
+/* }}} */
+
+
+/* Data modification fops - TYPE I */
+
+/* {{{ */
+
+/* {f}truncate() */
+
+int32_t
+changelog_truncate_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this, int32_t op_ret,
+                        int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (truncate, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_truncate (call_frame_t *frame,
+                    xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_truncate_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
+                    loc, offset, xdata);
+        return 0;
+}
+
+int32_t
+changelog_ftruncate_cbk (call_frame_t *frame,
+                         void *cookie, xlator_t *this, int32_t op_ret,
+                         int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (ftruncate, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_ftruncate (call_frame_t *frame,
+                     xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_ftruncate_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
+                    fd, offset, xdata);
+        return 0;
+}
+
+/* writev() */
+
+int32_t
+changelog_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf,
+                    dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret <= 0) || !local), unwind);
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (writev, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_writev (call_frame_t *frame,
+                  xlator_t *this, fd_t *fd, struct iovec *vector,
+                  int32_t count, off_t offset, uint32_t flags,
+                  struct iobref *iobref, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+
+ wind:
+        STACK_WIND (frame, changelog_writev_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev, fd, vector,
+                    count, offset, flags, iobref, xdata);
+        return 0;
+}
+
+/* }}} */
+
+/**
+ * The
+ *   - @init ()
+ *   - @fini ()
+ *   - @reconfigure ()
+ *   ... and helper routines
+ */
+
+/**
+ * needed if there are more operation modes in the future.
+ */
+static void
+changelog_assign_opmode (changelog_priv_t *priv, char *mode)
+{
+        if ( strncmp (mode, "realtime", 8) == 0 ) {
+                priv->op_mode = CHANGELOG_MODE_RT;
+        }
+}
+
+static void
+changelog_assign_encoding (changelog_priv_t *priv, char *enc)
+{
+        if ( strncmp (enc, "binary", 6) == 0 ) {
+                priv->encode_mode = CHANGELOG_ENCODE_BINARY;
+        } else if ( strncmp (enc, "ascii", 5) == 0 ) {
+                priv->encode_mode = CHANGELOG_ENCODE_ASCII;
+        }
+}
+
+/* cleanup any helper threads that are running */
+static void
+changelog_cleanup_helper_threads (xlator_t *this, changelog_priv_t *priv)
+{
+        if (priv->cr.rollover_th) {
+                changelog_thread_cleanup (this, priv->cr.rollover_th);
+                priv->cr.rollover_th = 0;
+        }
+
+        if (priv->cf.fsync_th) {
+                changelog_thread_cleanup (this, priv->cf.fsync_th);
+                priv->cf.fsync_th = 0;
+        }
+}
+
+/* spawn helper thread; cleaning up in case of errors */
+static int
+changelog_spawn_helper_threads (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret = 0;
+
+        priv->cr.this = this;
+        ret = gf_thread_create (&priv->cr.rollover_th,
+				NULL, changelog_rollover, priv);
+        if (ret)
+                goto out;
+
+        if (priv->fsync_interval) {
+                priv->cf.this = this;
+                ret = gf_thread_create (&priv->cf.fsync_th,
+					NULL, changelog_fsync_thread, priv);
+        }
+
+        if (ret)
+                changelog_cleanup_helper_threads (this, priv);
+
+ out:
+        return ret;
+}
+
+/* cleanup the notifier thread */
+static int
+changelog_cleanup_notifier (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret = 0;
+
+        if (priv->cn.notify_th) {
+                changelog_thread_cleanup (this, priv->cn.notify_th);
+                priv->cn.notify_th = 0;
+
+                ret = close (priv->wfd);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error closing writer end of notifier pipe"
+                                " (reason: %s)", strerror (errno));
+        }
+
+        return ret;
+}
+
+/* spawn the notifier thread - nop if already running */
+static int
+changelog_spawn_notifier (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret        = 0;
+        int flags      = 0;
+        int pipe_fd[2] = {0, 0};
+
+        if (priv->cn.notify_th)
+                goto out; /* notifier thread already running */
+
+        ret = pipe (pipe_fd);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot create pipe (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        /* writer is non-blocking */
+        flags = fcntl (pipe_fd[1], F_GETFL);
+        flags |= O_NONBLOCK;
+
+        ret = fcntl (pipe_fd[1], F_SETFL, flags);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set O_NONBLOCK flag");
+                goto out;
+        }
+
+        priv->wfd = pipe_fd[1];
+
+        priv->cn.this = this;
+        priv->cn.rfd  = pipe_fd[0];
+
+        ret = gf_thread_create (&priv->cn.notify_th,
+				NULL, changelog_notifier, priv);
+
+ out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_changelog_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
+                        " init failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+static int
+changelog_init (xlator_t *this, changelog_priv_t *priv)
+{
+        int                  i   = 0;
+        int                  ret = -1;
+        struct timeval       tv  = {0,};
+        changelog_log_data_t cld = {0,};
+
+        ret = gettimeofday (&tv, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gettimeofday() failure");
+                goto out;
+        }
+
+        priv->slice.tv_start = tv;
+
+        priv->maps[CHANGELOG_TYPE_DATA]     = "D ";
+        priv->maps[CHANGELOG_TYPE_METADATA] = "M ";
+        priv->maps[CHANGELOG_TYPE_ENTRY]    = "E ";
+
+        for (; i < CHANGELOG_MAX_TYPE; i++) {
+                /* start with version 1 */
+                priv->slice.changelog_version[i] = 1;
+        }
+
+        if (!priv->active)
+                return ret;
+
+        /* spawn the notifier thread */
+        ret = changelog_spawn_notifier (this, priv);
+        if (ret)
+                goto out;
+
+        /**
+         * start with a fresh changelog file every time. this is done
+         * in case there was an encoding change. so... things are kept
+         * simple here.
+         */
+        ret = changelog_fill_rollover_data (&cld, _gf_false);
+        if (ret)
+                goto out;
+
+        LOCK (&priv->lock);
+        {
+                ret = changelog_inject_single_event (this, priv, &cld);
+        }
+        UNLOCK (&priv->lock);
+
+        /* ... and finally spawn the helpers threads */
+        ret = changelog_spawn_helper_threads (this, priv);
+
+ out:
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int                     ret            = 0;
+        char                   *tmp            = NULL;
+        changelog_priv_t       *priv           = NULL;
+        gf_boolean_t            active_earlier = _gf_true;
+        gf_boolean_t            active_now     = _gf_true;
+        changelog_time_slice_t *slice          = NULL;
+        changelog_log_data_t    cld            = {0,};
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+
+        ret = -1;
+        active_earlier = priv->active;
+
+        /* first stop the rollover and the fsync thread */
+        changelog_cleanup_helper_threads (this, priv);
+
+        GF_OPTION_RECONF ("changelog-dir", tmp, options, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-dir\" option is not set");
+                goto out;
+        }
+
+        GF_FREE (priv->changelog_dir);
+        priv->changelog_dir = gf_strdup (tmp);
+        if (!priv->changelog_dir)
+                goto out;
+
+        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+        if (ret)
+                goto out;
+
+        GF_OPTION_RECONF ("changelog", active_now, options, bool, out);
+
+        /**
+         * changelog_handle_change() handles changes that could possibly
+         * have been submit changes before changelog deactivation.
+         */
+        if (!active_now)
+                priv->active = _gf_false;
+
+        GF_OPTION_RECONF ("op-mode", tmp, options, str, out);
+        changelog_assign_opmode (priv, tmp);
+
+        tmp = NULL;
+
+        GF_OPTION_RECONF ("encoding", tmp, options, str, out);
+        changelog_assign_encoding (priv, tmp);
+
+        GF_OPTION_RECONF ("rollover-time",
+                          priv->rollover_time, options, int32, out);
+        GF_OPTION_RECONF ("fsync-interval",
+                          priv->fsync_interval, options, int32, out);
+
+        if (active_now || active_earlier) {
+                ret = changelog_fill_rollover_data (&cld, !active_now);
+                if (ret)
+                        goto out;
+
+                slice = &priv->slice;
+
+                LOCK (&priv->lock);
+                {
+                        ret = changelog_inject_single_event (this, priv, &cld);
+                        if (!ret && active_now)
+                                SLICE_VERSION_UPDATE (slice);
+                }
+                UNLOCK (&priv->lock);
+
+                if (ret)
+                        goto out;
+
+                if (active_now) {
+                        ret = changelog_spawn_notifier (this, priv);
+                        if (!ret)
+                                ret = changelog_spawn_helper_threads (this,
+                                                                      priv);
+                } else
+                        ret = changelog_cleanup_notifier (this, priv);
+        }
+
+ out:
+        if (ret) {
+                ret = changelog_cleanup_notifier (this, priv);
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "changelog reconfigured");
+                if (active_now)
+                        priv->active = _gf_true;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        int               ret  = -1;
+        char             *tmp  = NULL;
+        changelog_priv_t *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO ("changelog", this, out);
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "translator needs a single subvolume");
+                goto out;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "dangling volume. please check volfile");
+                goto out;
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        this->local_pool = mem_pool_new (changelog_local_t, 64);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local memory pool");
+                goto out;
+        }
+
+        LOCK_INIT (&priv->lock);
+
+        GF_OPTION_INIT ("changelog-brick", tmp, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-brick\" option is not set");
+                goto out;
+        }
+
+        priv->changelog_brick = gf_strdup (tmp);
+        if (!priv->changelog_brick)
+                goto out;
+        tmp = NULL;
+
+        GF_OPTION_INIT ("changelog-dir", tmp, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-dir\" option is not set");
+                goto out;
+        }
+
+        priv->changelog_dir = gf_strdup (tmp);
+        if (!priv->changelog_dir)
+                goto out;
+        tmp = NULL;
+
+        /**
+         * create the directory even if change-logging would be inactive
+         * so that consumers can _look_ into it (finding nothing...)
+         */
+        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+        if (ret)
+                goto out;
+
+        GF_OPTION_INIT ("changelog", priv->active, bool, out);
+
+        GF_OPTION_INIT ("op-mode", tmp, str, out);
+        changelog_assign_opmode (priv, tmp);
+
+        tmp = NULL;
+
+        GF_OPTION_INIT ("encoding", tmp, str, out);
+        changelog_assign_encoding (priv, tmp);
+
+        GF_OPTION_INIT ("rollover-time", priv->rollover_time, int32, out);
+
+        GF_OPTION_INIT ("fsync-interval", priv->fsync_interval, int32, out);
+
+        changelog_encode_change(priv);
+
+        GF_ASSERT (cb_bootstrap[priv->op_mode].mode == priv->op_mode);
+        priv->cb = &cb_bootstrap[priv->op_mode];
+
+        /* ... now bootstrap the logger */
+        ret = priv->cb->ctor (this, &priv->cd);
+        if (ret)
+                goto out;
+
+        priv->changelog_fd = -1;
+        ret = changelog_init (this, priv);
+        if (ret)
+                goto out;
+
+        gf_log (this->name, GF_LOG_DEBUG, "changelog translator loaded");
+
+ out:
+        if (ret) {
+                if (this->local_pool)
+                        mem_pool_destroy (this->local_pool);
+                if (priv->cb) {
+                        ret = priv->cb->dtor (this, &priv->cd);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "error in cleanup during init()");
+                }
+                GF_FREE (priv->changelog_brick);
+                GF_FREE (priv->changelog_dir);
+                GF_FREE (priv);
+                this->private = NULL;
+        } else
+                this->private = priv;
+
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        int               ret  = -1;
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+
+        if (priv) {
+                ret = priv->cb->dtor (this, &priv->cd);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error in fini");
+                mem_pool_destroy (this->local_pool);
+                GF_FREE (priv->changelog_brick);
+                GF_FREE (priv->changelog_dir);
+                GF_FREE (priv);
+        }
+
+        this->private = NULL;
+
+        return;
+}
+
+struct xlator_fops fops = {
+        .mknod        = changelog_mknod,
+        .mkdir        = changelog_mkdir,
+        .create       = changelog_create,
+        .symlink      = changelog_symlink,
+        .writev       = changelog_writev,
+        .truncate     = changelog_truncate,
+        .ftruncate    = changelog_ftruncate,
+        .link         = changelog_link,
+        .rename       = changelog_rename,
+        .unlink       = changelog_unlink,
+        .rmdir        = changelog_rmdir,
+        .setattr      = changelog_setattr,
+        .fsetattr     = changelog_fsetattr,
+        .setxattr     = changelog_setxattr,
+        .fsetxattr    = changelog_fsetxattr,
+        .removexattr  = changelog_removexattr,
+        .fremovexattr = changelog_fremovexattr,
+};
+
+struct xlator_cbks cbks = {
+        .forget = changelog_forget,
+};
+
+struct volume_options options[] = {
+        {.key = {"changelog"},
+         .type = GF_OPTION_TYPE_BOOL,
+         .default_value = "off",
+         .description = "enable/disable change-logging"
+        },
+        {.key = {"changelog-brick"},
+         .type = GF_OPTION_TYPE_PATH,
+         .description = "brick path to generate unique socket file name."
+                       " should be the export directory of the volume strictly."
+        },
+        {.key = {"changelog-dir"},
+         .type = GF_OPTION_TYPE_PATH,
+         .description = "directory for the changelog files"
+        },
+        {.key = {"op-mode"},
+         .type = GF_OPTION_TYPE_STR,
+         .default_value = "realtime",
+         .value = {"realtime"},
+         .description = "operation mode - futuristic operation modes"
+        },
+        {.key = {"encoding"},
+         .type = GF_OPTION_TYPE_STR,
+         .default_value = "ascii",
+         .value = {"binary", "ascii"},
+         .description = "encoding type for changelogs"
+        },
+        {.key = {"rollover-time"},
+         .default_value = "60",
+         .type = GF_OPTION_TYPE_TIME,
+         .description = "time to switch to a new changelog file (in seconds)"
+        },
+        {.key = {"fsync-interval"},
+         .type = GF_OPTION_TYPE_TIME,
+         .default_value = "0",
+         .description = "do not open CHANGELOG file with O_SYNC mode."
+                        " instead perform fsync() at specified intervals"
+        },
+        {.key = {NULL}
+        },
+};
diff --git a/xlators/features/compress/Makefile.am b/xlators/features/compress/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/compress/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/compress/src/Makefile.am b/xlators/features/compress/src/Makefile.am
new file mode 100644
index 000000000..4a64b52a9
--- /dev/null
+++ b/xlators/features/compress/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = cdc.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = cdc.h cdc-mem-types.h
+
+cdc_la_LDFLAGS = -module -avoidversion $(LIBZ_LIBS)
+
+cdc_la_SOURCES = cdc.c cdc-helper.c
+cdc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
+-shared -nostartfiles  $(LIBZ_CFLAGS)
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/compress/src/cdc-helper.c b/xlators/features/compress/src/cdc-helper.c
new file mode 100644
index 000000000..54432ff45
--- /dev/null
+++ b/xlators/features/compress/src/cdc-helper.c
@@ -0,0 +1,547 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#ifdef HAVE_LIB_Z
+/* gzip header looks something like this
+ * (RFC 1950)
+ *
+ * +---+---+---+---+---+---+---+---+---+---+
+ * |ID1|ID2|CM |FLG|     MTIME     |XFL|OS |
+ * +---+---+---+---+---+---+---+---+---+---+
+ *
+ * Data is usually sent without this header i.e
+ * Data sent = <compressed-data> + trailer(8)
+ * The trailer contains the checksum.
+ *
+ * gzip_header is added only during debugging.
+ * Refer to the function cdc_dump_iovec_to_disk
+ */
+static const char gzip_header[10] =
+        {
+                '\037', '\213', Z_DEFLATED, 0,
+                0,      0,      0,          0,
+                0,      GF_CDC_OS_ID
+        };
+
+static int32_t
+cdc_next_iovec (xlator_t *this, cdc_info_t *ci)
+{
+        int ret = -1;
+
+        ci->ncount++;
+        /* check for iovec overflow -- should not happen */
+        if (ci->ncount == MAX_IOVEC) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib output buffer overflow"
+                        " ->ncount (%d) | ->MAX_IOVEC (%d)",
+                        ci->ncount, MAX_IOVEC);
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_put_long (unsigned char *string, unsigned long x)
+{
+        string[0] = (unsigned char) (x & 0xff);
+        string[1] = (unsigned char) ((x & 0xff00) >> 8);
+        string[2] = (unsigned char) ((x & 0xff0000) >> 16);
+        string[3] = (unsigned char) ((x & 0xff000000) >> 24);
+}
+
+static unsigned long
+cdc_get_long (unsigned char *buf)
+{
+        return ((unsigned long) buf[0])
+                | (((unsigned long) buf[1]) << 8)
+                | (((unsigned long) buf[2]) << 16)
+                | (((unsigned long) buf[3]) << 24);
+}
+
+static int32_t
+cdc_init_gzip_trailer (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int   ret = -1;
+        char *buf = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        buf = CURR_VEC(ci).iov_base =
+                (char *) GF_CALLOC (1, GF_CDC_VALIDATION_SIZE,
+                                    gf_cdc_mt_gzip_trailer_t);
+
+        if (!CURR_VEC(ci).iov_base)
+                goto out;
+
+        CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
+
+        cdc_put_long ((unsigned char *)&buf[0], ci->crc);
+        cdc_put_long ((unsigned char *)&buf[4], ci->stream.total_in);
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static int32_t
+cdc_alloc_iobuf_and_init_vec (xlator_t *this,
+                              cdc_priv_t *priv, cdc_info_t *ci,
+                              int size)
+{
+        int           ret       = -1;
+        int           alloc_len = 0;
+        struct iobuf *iobuf     = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        alloc_len = size ? size : ci->buffer_size;
+
+        iobuf = iobuf_get2 (this->ctx->iobuf_pool, alloc_len);
+        if (!iobuf)
+                goto out;
+
+        ret = iobref_add (ci->iobref, iobuf);
+        if (ret)
+                goto out;
+
+        /* Initialize this iovec */
+        CURR_VEC(ci).iov_base = iobuf->ptr;
+        CURR_VEC(ci).iov_len  = alloc_len;
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
+{
+        ci->stream.next_out  = (unsigned char *) CURR_VEC(ci).iov_base;
+        ci->stream.avail_out = size ? size : ci->buffer_size;
+}
+
+/* This routine is for testing and debugging only.
+ * Data written = header(10) + <compressed-data> + trailer(8)
+ * So each gzip dump file is at least 18 bytes in size.
+ */
+void
+cdc_dump_iovec_to_disk (xlator_t *this, cdc_info_t *ci, const char *file)
+{
+        int    i      = 0;
+        int    fd     = 0;
+        size_t writen = 0;
+        size_t total_writen = 0;
+
+        fd = open (file, O_WRONLY|O_CREAT|O_TRUNC, 0777 );
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot open file: %s", file);
+                return;
+        }
+
+        writen = write (fd, (char *) gzip_header, 10);
+        total_writen += writen;
+        for (i = 0; i < ci->ncount; i++) {
+                writen = write (fd, (char *) ci->vec[i].iov_base, ci->vec[i].iov_len);
+                total_writen += writen;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                        "dump'd %zu bytes to %s", total_writen, GF_CDC_DEBUG_DUMP_FILE );
+
+        close (fd);
+}
+
+static int32_t
+cdc_flush_libz_buffer (cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
+                       int (*libz_func)(z_streamp, int),
+                       int flush)
+{
+        int32_t ret = Z_OK;
+        int done = 0;
+        unsigned int deflate_len = 0;
+
+        for (;;) {
+                deflate_len = ci->buffer_size - ci->stream.avail_out;
+
+                if (deflate_len != 0) {
+                        CURR_VEC(ci).iov_len = deflate_len;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret) {
+                                ret = Z_MEM_ERROR;
+                                break;
+                        }
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                if (done) {
+                        ci->ncount--;
+                        break;
+                }
+
+                ret = libz_func (&ci->stream, flush);
+
+                if (ret == Z_BUF_ERROR) {
+                        ret = Z_OK;
+                        ci->ncount--;
+                        break;
+                }
+
+                done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
+
+                if (ret != Z_OK && ret != Z_STREAM_END)
+                        break;
+        }
+
+        return ret;
+}
+
+static int32_t
+do_cdc_compress (struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
+                 cdc_info_t *ci)
+{
+        int ret = -1;
+
+        /* Initialize defalte */
+        ret = deflateInit2 (&ci->stream, priv->cdc_level, Z_DEFLATED,
+                            priv->window_size, priv->mem_level,
+                            Z_DEFAULT_STRATEGY);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to init Zlib (retval: %d)", ret);
+                goto out;
+        }
+
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in  = (unsigned char *) vec->iov_base;
+        ci->stream.avail_in = vec->iov_len;
+
+        ci->crc = crc32 (ci->crc, (const Bytef *) vec->iov_base, vec->iov_len);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d",
+                ci->crc, ci->stream.avail_in, ci->buffer_size);
+
+        /* compress !! */
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = deflate (&ci->stream, Z_NO_FLUSH);
+                if (ret != Z_OK)
+                        break;
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_compress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+              dict_t **xdata)
+{
+        int ret = -1;
+        int i   = 0;
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto out;
+
+        if (!*xdata) {
+                *xdata = dict_new ();
+                if (!*xdata) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot allocate xdata"
+                                " dict");
+                        goto out;
+                }
+        }
+
+        /* data */
+        for (i = 0; i < ci->count; i++) {
+                ret = do_cdc_compress (&ci->vector[i], this, priv, ci);
+                if (ret != Z_OK)
+                        goto deflate_cleanup_out;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, deflate, Z_FINISH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Compression Error: ret (%d)", ret);
+                ret = -1;
+                goto deflate_cleanup_out;
+        }
+
+        /* trailer */
+        ret = cdc_init_gzip_trailer (this, priv, ci);
+        if (ret)
+                goto deflate_cleanup_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Compressed %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+        ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
+
+        /* set deflated canary value for identification */
+        ret = dict_set_int32 (*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
+        if (ret) {
+                /* Send uncompressed data if we can't _tell_ the client
+                 * that deflated data is on it's way. So, we just log
+                 * the faliure and continue as usual.
+                 */
+                 gf_log (this->name, GF_LOG_ERROR,
+                 "Data deflated, but could not set canary"
+                 " value in dict for identification");
+        }
+
+        /* This is to be used in testing */
+        if ( priv->debug ) {
+                cdc_dump_iovec_to_disk (this, ci, GF_CDC_DEBUG_DUMP_FILE );
+        }
+
+ deflate_cleanup_out:
+        (void) deflateEnd(&ci->stream);
+
+ out:
+        return ret;
+}
+
+
+/* deflate content is checked by the presence of a canary
+ * value in the dict as the key
+ */
+static int32_t
+cdc_check_content_for_deflate (dict_t *xdata)
+{
+        return dict_get (xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
+}
+
+static unsigned long
+cdc_extract_crc (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[0]);
+}
+
+static unsigned long
+cdc_extract_size (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[4]);
+}
+
+static int32_t
+cdc_validate_inflate (cdc_info_t *ci, unsigned long crc,
+                      unsigned long len)
+{
+        return !((crc == ci->crc)
+                 /* inflated length is hidden inside
+                  * Zlib stream struct */
+                 && (len == ci->stream.total_out));
+}
+
+static int32_t
+do_cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int            ret          = -1;
+        int            i            = 0;
+        int            len          = 0;
+        char          *inflte       = NULL;
+        char          *trailer      = NULL;
+        struct iovec   vec          = {0,};
+        unsigned long  computed_crc = 0;
+        unsigned long  computed_len = 0;
+
+        ret = inflateInit2 (&ci->stream, priv->window_size);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib: Unable to initialize inflate");
+                goto out;
+        }
+
+        vec = THIS_VEC(ci, 0);
+
+        trailer = (char *) (((char *) vec.iov_base) + vec.iov_len
+                - GF_CDC_VALIDATION_SIZE);
+
+        /* CRC of uncompressed data */
+        computed_crc = cdc_extract_crc (trailer);
+
+        /* size of uncomrpessed data */
+        computed_len = cdc_extract_size (trailer);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
+                computed_crc, computed_len, ci->buffer_size);
+
+        inflte = vec.iov_base ;
+        len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
+
+        /* allocate buffer of the original length of the data */
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in = (unsigned char *) inflte;
+        ci->stream.avail_in = len;
+
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = inflate (&ci->stream, Z_NO_FLUSH);
+                if (ret == Z_STREAM_ERROR)
+                        break;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, inflate, Z_SYNC_FLUSH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Decompression Error: ret (%d)", ret);
+                ret = -1;
+                goto out;
+        }
+
+        /* compute CRC of the uncompresses data to check for
+         * correctness */
+
+        for (i = 0; i < ci->ncount; i++) {
+                ci->crc = crc32 (ci->crc,
+                                 (const Bytef *) ci->vec[i].iov_base,
+                                 ci->vec[i].iov_len);
+        }
+
+        /* validate inflated data */
+        ret = cdc_validate_inflate (ci, computed_crc, computed_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Checksum or length mismatched in inflated data");
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+                dict_t *xdata)
+{
+        int32_t ret = -1;
+
+        /* check for deflate content */
+        if (!cdc_check_content_for_deflate (xdata)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Content not deflated, passing through ...");
+                goto passthrough_out;
+        }
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto passthrough_out;
+
+        /* do we need to do this? can we assume that one iovec
+         * will hold per request data everytime?
+         *
+         * server/client protocol seems to deal with a single
+         * iovec even if op_ret > 1M. So, it looks ok to
+         * assume that a single iovec will contain all the
+         * data (This saves us a lot from finding the trailer
+         * and the data since it could have been split-up onto
+         * two adjacent iovec's.
+         *
+         * But, in case this translator is loaded above quick-read
+         * for some reason, then it's entirely possible that we get
+         * multiple iovec's...
+         *
+         * This case (handled below) is not tested. (by loading the
+         * xlator below quick-read)
+         */
+
+        /* @@ I_HOPE_THIS_IS_NEVER_HIT */
+        if (ci->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING, "unable to handle"
+                        " multiple iovecs (%d in number)", ci->count);
+                goto inflate_cleanup_out;
+                /* TODO: coallate all iovecs in one */
+        }
+
+        ret = do_cdc_decompress (this, priv, ci);
+        if (ret)
+                goto inflate_cleanup_out;
+
+        ci->nbytes = ci->stream.total_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Inflated %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+ inflate_cleanup_out:
+        (void) inflateEnd (&ci->stream);
+
+ passthrough_out:
+        return ret;
+}
+
+#endif
diff --git a/xlators/features/compress/src/cdc-mem-types.h b/xlators/features/compress/src/cdc-mem-types.h
new file mode 100644
index 000000000..efa008059
--- /dev/null
+++ b/xlators/features/compress/src/cdc-mem-types.h
@@ -0,0 +1,22 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_MEM_TYPES_H
+#define __CDC_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_cdc_mem_types {
+        gf_cdc_mt_priv_t         = gf_common_mt_end + 1,
+        gf_cdc_mt_vec_t          = gf_common_mt_end + 2,
+        gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
+};
+
+#endif
diff --git a/xlators/features/compress/src/cdc.c b/xlators/features/compress/src/cdc.c
new file mode 100644
index 000000000..eb7d87c56
--- /dev/null
+++ b/xlators/features/compress/src/cdc.c
@@ -0,0 +1,342 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+static void
+cdc_cleanup_iobref (cdc_info_t *ci)
+{
+        assert(ci->iobref != NULL);
+        iobref_clear (ci->iobref);
+}
+
+int32_t
+cdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iovec *vector, int32_t count,
+               struct iatt *stbuf, struct iobref *iobref,
+               dict_t *xdata)
+{
+        int         ret   = -1;
+        cdc_priv_t *priv  = NULL;
+        cdc_info_t  ci    = {0,};
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+        priv = this->private;
+
+        if (op_ret <= 0)
+                goto default_out;
+
+        if ( (priv->min_file_size != 0)
+             && (op_ret < priv->min_file_size) )
+                goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = op_ret;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A readv compresses on the server side and decompresses on the client side
+ */
+        if (priv->op_mode == GF_CDC_MODE_SERVER) {
+                ret = cdc_compress (this, priv, &ci, &xdata);
+        } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+                ret = cdc_decompress (this, priv, &ci, xdata);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid operation mode (%d)", priv->op_mode);
+        }
+
+        if (ret)
+                goto default_out;
+
+        STACK_UNWIND_STRICT (readv, frame, ci.nbytes, op_errno,
+                             ci.vec, ci.ncount, stbuf, iobref,
+                             xdata);
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
+                             vector, count, stbuf, iobref, xdata);
+        return 0;
+}
+
+int32_t
+cdc_readv (call_frame_t *frame, xlator_t *this,
+           fd_t *fd, size_t size, off_t offset, uint32_t flags,
+           dict_t *xdata)
+{
+        fop_readv_cbk_t cbk = NULL;
+
+#ifdef HAVE_LIB_Z
+        cbk = cdc_readv_cbk;
+#else
+        cbk = default_readv_cbk;
+#endif
+        STACK_WIND (frame, cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+        return 0;
+}
+
+int32_t
+cdc_writev_cbk (call_frame_t *frame,
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+	return 0;
+}
+
+int32_t
+cdc_writev (call_frame_t *frame,
+            xlator_t *this,
+            fd_t *fd,
+            struct iovec *vector,
+            int32_t count,
+            off_t offset,
+            uint32_t flags,
+            struct iobref *iobref, dict_t *xdata)
+{
+	int	     ret   = -1;
+	cdc_priv_t  *priv  = NULL;
+	cdc_info_t   ci    = {0,};
+	size_t       isize = 0;
+
+	GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+	GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+	priv = this->private;
+
+	isize =  iov_length(vector, count);
+
+	if (isize <= 0)
+                goto default_out;
+
+        if ( (priv->min_file_size != 0)
+             && (isize < priv->min_file_size) )
+                goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = isize;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A writev compresses on the client side and decompresses on the server side
+ */
+	if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+		ret = cdc_compress (this, priv, &ci, &xdata);
+	} else if (priv->op_mode == GF_CDC_MODE_SERVER) {
+		ret = cdc_decompress (this, priv, &ci, xdata);
+	} else {
+		gf_log (this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ", priv->op_mode);
+	}
+
+	if (ret)
+		goto default_out;
+
+	STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, ci.vec, ci.ncount, offset, flags,
+                    iobref, xdata);
+
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+	STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, vector, count, offset, flags,
+                    iobref, xdata);
+	return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        int         ret      = -1;
+        char       *temp_str = NULL;
+        cdc_priv_t *priv     = NULL;
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, err);
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Need subvolume == 1");
+                goto err;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Dangling volume. Check volfile");
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_cdc_mt_priv_t);
+        if (!priv) {
+                goto err;
+        }
+
+        /* Check if debug mode is turned on */
+        GF_OPTION_INIT ("debug", priv->debug, bool, err);
+        if( priv->debug ) {
+                gf_log (this->name, GF_LOG_DEBUG, "CDC debug option turned on");
+        }
+
+        /* Set Gzip Window Size */
+        GF_OPTION_INIT ("window-size", priv->window_size, int32, err);
+        if ( (priv->window_size > GF_CDC_MAX_WINDOWSIZE)
+             || (priv->window_size < GF_CDC_DEF_WINDOWSIZE) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip window size (%d), using default",
+                        priv->window_size);
+                priv->window_size = GF_CDC_DEF_WINDOWSIZE;
+        }
+
+        /* Set Gzip (De)Compression Level */
+        GF_OPTION_INIT ("compression-level", priv->cdc_level, int32, err);
+        if ( ((priv->cdc_level < 1) || (priv->cdc_level > 9))
+             && (priv->cdc_level != GF_CDC_DEF_COMPRESSION) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip (de)compression level (%d),"
+                        " using default", priv->cdc_level);
+                priv->cdc_level = GF_CDC_DEF_COMPRESSION;
+        }
+
+        /* Set Gzip Memory Level */
+        GF_OPTION_INIT ("mem-level", priv->mem_level, int32, err);
+        if ( (priv->mem_level < 1) || (priv->mem_level > 9) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip memory level, using the default");
+                priv->mem_level = GF_CDC_DEF_MEMLEVEL;
+        }
+
+        /* Set min file size to enable compression */
+        GF_OPTION_INIT ("min-size", priv->min_file_size, int32, err);
+
+        /* Mode of operation - Server/Client */
+        ret = dict_get_str (this->options, "mode", &temp_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Operation mode not specified !!");
+                goto err;
+        }
+
+        if (GF_CDC_MODE_IS_CLIENT (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_CLIENT;
+        } else if (GF_CDC_MODE_IS_SERVER (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_SERVER;
+        } else {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Bogus operation mode (%s) specified", temp_str);
+                goto err;
+        }
+
+        this->private = priv;
+        gf_log (this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",temp_str);
+        return 0;
+
+ err:
+        if (priv)
+                GF_FREE (priv);
+
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        cdc_priv_t *priv = this->private;
+
+        if (priv)
+                GF_FREE (priv);
+        this->private = NULL;
+        return;
+}
+
+struct xlator_fops fops = {
+        .readv  = cdc_readv,
+        .writev = cdc_writev,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {"window-size"},
+          .default_value = "-15",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Size of the zlib history buffer."
+        },
+        { .key  = {"mem-level"},
+          .default_value = "8",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Memory allocated for internal compression state.\
+                          1 uses minimum memory but is slow and reduces \
+                          compression ratio; memLevel=9 uses maximum memory \
+                          for optimal speed. The default value is 8."
+        },
+        { .key  = {"compression-level"},
+          .default_value = "-1",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Compression levels \
+                          0 : no compression, 1 : best speed, \
+                          9 : best compression, -1 : default compression "
+        },
+        { .key  = {"min-size"},
+          .default_value = "0",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Data is compressed only when its size exceeds this."
+        },
+        { .key  = {"mode"},
+          .value = {"server", "client"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Set on the basis of where the xlator is loaded."
+        },
+        { .key = {"debug"},
+          .default_value = "false",
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "This is used in testing. Will dump compressed data \
+                          to disk as a gzip file."
+        },
+        { .key  = {NULL}
+        },
+};
diff --git a/xlators/features/compress/src/cdc.h b/xlators/features/compress/src/cdc.h
new file mode 100644
index 000000000..71f4d2317
--- /dev/null
+++ b/xlators/features/compress/src/cdc.h
@@ -0,0 +1,107 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_H
+#define __CDC_H
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#include "xlator.h"
+
+#ifndef MAX_IOVEC
+#define MAX_IOVEC 16
+#endif
+
+typedef struct cdc_priv {
+        int window_size;
+        int mem_level;
+        int cdc_level;
+        int min_file_size;
+        int op_mode;
+        gf_boolean_t debug;
+        gf_lock_t lock;
+} cdc_priv_t;
+
+typedef struct cdc_info {
+        /* input bits */
+        int            count;
+        int32_t        ibytes;
+        struct iovec  *vector;
+        struct iatt   *buf;
+
+        /* output bits */
+        int            ncount;
+        int            nbytes;
+        int            buffer_size;
+        struct iovec   vec[MAX_IOVEC];
+        struct iobref *iobref;
+
+        /* zlib bits */
+#ifdef HAVE_LIB_Z
+        z_stream      stream;
+#endif
+        unsigned long crc;
+} cdc_info_t;
+
+#define NVEC(ci) (ci->ncount - 1)
+#define CURR_VEC(ci) ci->vec[ci->ncount - 1]
+#define THIS_VEC(ci, i) ci->vector[i]
+
+/* Gzip defaults */
+#define GF_CDC_DEF_WINDOWSIZE  -15 /* default value */
+#define GF_CDC_MAX_WINDOWSIZE  -8  /* max value     */
+
+#ifdef HAVE_LIB_Z
+#define GF_CDC_DEF_COMPRESSION Z_DEFAULT_COMPRESSION
+#else
+#define GF_CDC_DEF_COMPRESSION -1
+#endif
+
+#define GF_CDC_DEF_MEMLEVEL    8
+#define GF_CDC_DEF_BUFFERSIZE  262144 // 256K - default compression buffer size
+
+/* Operation mode
+ * If xlator is loaded on client, readv decompresses and writev compresses
+ * If xlator is loaded on server, readv compresses and writev decompresses
+ */
+#define GF_CDC_MODE_CLIENT   0
+#define GF_CDC_MODE_SERVER   1
+
+/* min size of data to do cmpression
+ * 0 == compress even 1byte
+ */
+#define GF_CDC_MIN_CHUNK_SIZE 0
+
+#define GF_CDC_VALIDATION_SIZE 8
+
+#define GF_CDC_OS_ID 0xFF
+#define GF_CDC_DEFLATE_CANARY_VAL "deflate"
+#define GF_CDC_DEBUG_DUMP_FILE "/tmp/cdcdump.gz"
+
+#define GF_CDC_MODE_IS_CLIENT(m) \
+        (strcmp (m, "client") == 0)
+
+#define GF_CDC_MODE_IS_SERVER(m) \
+        (strcmp (m, "server") == 0)
+
+int32_t
+cdc_compress (xlator_t *this,
+              cdc_priv_t *priv,
+              cdc_info_t *ci,
+              dict_t **xdata);
+int32_t
+cdc_decompress (xlator_t *this,
+                cdc_priv_t *priv,
+                cdc_info_t *ci,
+                dict_t *xdata);
+
+#endif
diff --git a/xlators/features/filter/src/Makefile.am b/xlators/features/filter/src/Makefile.am
index fa0b92214..d1fda8b0a 100644
--- a/xlators/features/filter/src/Makefile.am
+++ b/xlators/features/filter/src/Makefile.am
@@ -1,13 +1,16 @@
 xlator_LTLIBRARIES = filter.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-filter_la_LDFLAGS = -module -avoidversion
+filter_la_LDFLAGS = -module -avoid-version
 
 filter_la_SOURCES = filter.c
 filter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = filter-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/filter/src/filter-mem-types.h b/xlators/features/filter/src/filter-mem-types.h
new file mode 100644
index 000000000..47a17249b
--- /dev/null
+++ b/xlators/features/filter/src/filter-mem-types.h
@@ -0,0 +1,20 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __FILTER_MEM_TYPES_H__
+#define __FILTER_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_filter_mem_types_ {
+        gf_filter_mt_gf_filter = gf_common_mt_end + 1,
+        gf_filter_mt_end
+};
+#endif
+
diff --git a/xlators/features/filter/src/filter.c b/xlators/features/filter/src/filter.c
index 67ea45d3a..1d4887b71 100644
--- a/xlators/features/filter/src/filter.c
+++ b/xlators/features/filter/src/filter.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -30,6 +20,7 @@
 #include "logging.h"
 #include "dict.h"
 #include "xlator.h"
+#include "filter-mem-types.h"
 
 #define GF_FILTER_NOBODY_UID         65534
 #define GF_FILTER_NOBODY_GID         65534
@@ -159,36 +150,36 @@ update_frame (call_frame_t *frame,
 
 /* if 'root' don't change the uid/gid */
 static int32_t
-update_stat (struct stat *stbuf,
+update_stat (struct iatt *stbuf,
 	     struct gf_filter *filter)
 {
 	int32_t idx = 0;
 	for (idx = 0; idx < filter->translate_num_uid_entries; idx++) {
-		if (stbuf->st_uid == GF_FILTER_ROOT_UID)
+		if (stbuf->ia_uid == GF_FILTER_ROOT_UID)
 			continue;
-		if ((stbuf->st_uid >= filter->translate_input_uid[idx][0]) &&
-		    (stbuf->st_uid <= filter->translate_input_uid[idx][1])) {
-			stbuf->st_uid = filter->translate_output_uid[idx];
+		if ((stbuf->ia_uid >= filter->translate_input_uid[idx][0]) &&
+		    (stbuf->ia_uid <= filter->translate_input_uid[idx][1])) {
+			stbuf->ia_uid = filter->translate_output_uid[idx];
 			break;
 		}
 	}
 	
 	for (idx = 0; idx < filter->translate_num_gid_entries; idx++) {
-		if (stbuf->st_gid == GF_FILTER_ROOT_GID)
+		if (stbuf->ia_gid == GF_FILTER_ROOT_GID)
 			continue;
-		if ((stbuf->st_gid >= filter->translate_input_gid[idx][0]) &&
-		    (stbuf->st_gid <= filter->translate_input_gid[idx][1])) {
-			stbuf->st_gid = filter->translate_output_gid[idx];
+		if ((stbuf->ia_gid >= filter->translate_input_gid[idx][0]) &&
+		    (stbuf->ia_gid <= filter->translate_input_gid[idx][1])) {
+			stbuf->ia_gid = filter->translate_output_gid[idx];
 			break;
 		}
 	}
 
 	if (filter->fixed_uid_set) {
-		stbuf->st_uid = filter->fixed_uid;
+		stbuf->ia_uid = filter->fixed_uid;
 	}
 
 	if (filter->fixed_gid_set) {
-		stbuf->st_gid = filter->fixed_gid;
+		stbuf->ia_gid = filter->fixed_gid;
 	}
 	
 	return 0;
@@ -201,19 +192,22 @@ filter_lookup_cbk (call_frame_t *frame,
 		   int32_t op_ret,
 		   int32_t op_errno,
 		   inode_t *inode,
-		   struct stat *buf,
-		   dict_t *dict)
+		   struct iatt *buf,
+		   dict_t *dict,
+                   struct iatt *postparent)
 {
 	int ret = 0;
 	if (op_ret >= 0) {
 		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->st_uid);
+		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
 		if (ret == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"couldn't set context");
 		}
+
+		update_stat (postparent, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, dict);
+	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, dict, postparent);
 	return 0;
 }
 
@@ -239,7 +233,7 @@ filter_stat_cbk (call_frame_t *frame,
 		 xlator_t *this,
 		 int32_t op_ret,
 		 int32_t op_errno,
-		 struct stat *buf)
+		 struct iatt *buf)
 {
 	if (op_ret >= 0) {
 		update_stat (buf, this->private);
@@ -262,25 +256,28 @@ filter_stat (call_frame_t *frame,
 }
 
 static int32_t
-filter_chmod_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct stat *buf)
+filter_setattr_cbk (call_frame_t *frame,
+                    void *cookie,
+                    xlator_t *this,
+                    int32_t op_ret,
+                    int32_t op_errno,
+                    struct iatt *preop,
+                    struct iatt *postop)
 {
 	if (op_ret >= 0) {
-		update_stat (buf, this->private);
+		update_stat (preop, this->private);
+		update_stat (postop, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
+	STACK_UNWIND (frame, op_ret, op_errno, preop, postop);
 	return 0;
 }
 
 int32_t
-filter_chmod (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      mode_t mode)
+filter_setattr (call_frame_t *frame,
+                xlator_t *this,
+                loc_t *loc,
+                struct iatt *stbuf,
+                int32_t valid)
 {
 	int32_t ret = 0;
 	ret = update_frame (frame, loc->inode, this->private);
@@ -291,147 +288,65 @@ filter_chmod (call_frame_t *frame,
 	case GF_FILTER_MAP_BOTH:
 		if (loc->inode->st_mode & S_IWOTH)
 			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL);
+		gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: returning permission denied", loc->path);
+		STACK_UNWIND (frame, -1, EPERM, NULL, NULL, NULL);
 		return 0;
-		
+
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL);
+		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
 		return 0;
 	default:
 		break;
 	}
 
 	STACK_WIND (frame,
-		    filter_chmod_cbk,
+		    filter_setattr_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->chmod,
+		    FIRST_CHILD(this)->fops->setattr,
 		    loc,
-		    mode);
+		    stbuf, valid);
 	return 0;
 }
 
-
 static int32_t
-filter_fchmod_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct stat *buf)
+filter_fsetattr_cbk (call_frame_t *frame,
+                     void *cookie,
+                     xlator_t *this,
+                     int32_t op_ret,
+                     int32_t op_errno,
+                     struct iatt *preop,
+                     struct iatt *postop)
 {
 	if (op_ret >= 0) {
-		update_stat (buf, this->private);
+                update_stat (preop, this->private);
+		update_stat (postop, this->private);
 	}
 	STACK_UNWIND (frame,
 		      op_ret,
 		      op_errno,
-		      buf);
+		      preop, postop);
 	return 0;
 }
 
-int32_t 
-filter_fchmod (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       mode_t mode)
+int32_t
+filter_fsetattr (call_frame_t *frame,
+                 xlator_t *this,
+                 fd_t *fd,
+                 struct iatt *stbuf,
+                 int32_t valid)
 {
 	STACK_WIND (frame,
-		    filter_fchmod_cbk,
+		    filter_fsetattr_cbk,
 		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fchmod,
+		    FIRST_CHILD(this)->fops->fsetattr,
 		    fd,
-		    mode);
-	return 0;
-}
-
-static int32_t
-filter_chown_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct stat *buf)
-{
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t
-filter_chown (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      uid_t uid,
-	      gid_t gid)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL);
-		return 0;
-		
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL);
-		return 0;
-	default:
-		break;
-	}			
-
-	STACK_WIND (frame,	      
-		    filter_chown_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->chown,
-		    loc,
-		    uid,
-		    gid);
+		    stbuf, valid);
 	return 0;
 }
 
-static int32_t
-filter_fchown_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct stat *buf)
-{
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-filter_fchown (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       uid_t uid,
-	       gid_t gid)
-{
-	STACK_WIND (frame,	      
-		    filter_fchown_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fchown,
-		    fd,
-		    uid,
-		    gid);
-	return 0;
-}
 
 static int32_t
 filter_truncate_cbk (call_frame_t *frame,
@@ -439,12 +354,14 @@ filter_truncate_cbk (call_frame_t *frame,
 		     xlator_t *this,
 		     int32_t op_ret,
 		     int32_t op_errno,
-		     struct stat *buf)
+		     struct iatt *prebuf,
+                     struct iatt *postbuf)
 {
 	if (op_ret >= 0) {
-		update_stat (buf, this->private);
+		update_stat (prebuf, this->private);
+		update_stat (postbuf, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
+	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
 	return 0;
 }
 
@@ -464,13 +381,13 @@ filter_truncate (call_frame_t *frame,
 		if (loc->inode->st_mode & S_IWOTH)
 			break;
 		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL);
+		STACK_UNWIND (frame, -1, EPERM, NULL, NULL);
 		return 0;
 		
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL);
+		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
 		return 0;
 	}			
 
@@ -489,12 +406,14 @@ filter_ftruncate_cbk (call_frame_t *frame,
 		      xlator_t *this,
 		      int32_t op_ret,
 		      int32_t op_errno,
-		      struct stat *buf)
+		      struct iatt *prebuf,
+                      struct iatt *postbuf)
 {
 	if (op_ret >= 0) {
-		update_stat (buf, this->private);
+		update_stat (prebuf, this->private);
+		update_stat (postbuf, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
+	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
 	return 0;
 }
 
@@ -513,56 +432,6 @@ filter_ftruncate (call_frame_t *frame,
 	return 0;
 }
 
-int32_t 
-filter_utimens_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct stat *buf)
-{
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-
-int32_t 
-filter_utimens (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		struct timespec tv[2])
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL);
-		return 0;
-		
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    filter_utimens_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->utimens,
-		    loc,
-		    tv);
-	return 0;
-}
 
 static int32_t
 filter_readlink_cbk (call_frame_t *frame,
@@ -570,9 +439,13 @@ filter_readlink_cbk (call_frame_t *frame,
 		     xlator_t *this,
 		     int32_t op_ret,
 		     int32_t op_errno,
-		     const char *path)
+		     const char *path,
+                     struct iatt *sbuf)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, path);
+        if (op_ret >= 0)
+                update_stat (sbuf, this->private);
+
+	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
 	return 0;
 }
 
@@ -612,19 +485,25 @@ filter_mknod_cbk (call_frame_t *frame,
 		  int32_t op_ret,
 		  int32_t op_errno,
 		  inode_t *inode,
-		  struct stat *buf)
+                  struct iatt *buf,
+                  struct iatt *preparent,
+                  struct iatt *postparent)
 {
 	int ret = 0;
 
 	if (op_ret >= 0) {
 		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->st_uid);
+		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
 		if (ret == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"couldn't set context");
 		}
+
+		update_stat (preparent, this->private);
+		update_stat (postparent, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
+	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
+                      preparent, postparent);
 	return 0;
 }
 
@@ -646,13 +525,15 @@ filter_mknod (call_frame_t *frame,
 		if (parent->st_mode & S_IWOTH)
 			break;
 		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
+		STACK_UNWIND (frame, -1, EPERM, NULL, NULL,
+                              NULL, NULL);
 		return 0;
 
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
+		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
+                              NULL, NULL);
 		return 0;
 	}
 	STACK_WIND (frame,
@@ -670,18 +551,24 @@ filter_mkdir_cbk (call_frame_t *frame,
 		  int32_t op_ret,
 		  int32_t op_errno,
 		  inode_t *inode,
-		  struct stat *buf)
+                  struct iatt *buf,
+                  struct iatt *preparent,
+                  struct iatt *postparent)
 {
 	int ret = 0;
 	if (op_ret >= 0) {
 		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->st_uid);
+		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
 		if (ret == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"couldn't set context");
 		}
+
+		update_stat (preparent, this->private);
+		update_stat (postparent, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
+	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
+                      preparent, postparent);
 	return 0;
 }
 
@@ -702,13 +589,15 @@ filter_mkdir (call_frame_t *frame,
 		if (parent->st_mode & S_IWOTH)
 			break;
 		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
+		STACK_UNWIND (frame, -1, EPERM, NULL, NULL,
+                              NULL, NULL);
 		return 0;
 
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
+		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
+                              NULL, NULL);
 		return 0;
 	}
 	STACK_WIND (frame,
@@ -724,9 +613,16 @@ filter_unlink_cbk (call_frame_t *frame,
 		   void *cookie,
 		   xlator_t *this,
 		   int32_t op_ret,
-		   int32_t op_errno)
+		   int32_t op_errno,
+                   struct iatt *preparent,
+                   struct iatt *postparent)
 {
-	STACK_UNWIND (frame, op_ret, op_errno);
+        if (op_ret >= 0) {
+		update_stat (preparent, this->private);
+		update_stat (postparent, this->private);
+        }
+
+	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
 	return 0;
 }
 
@@ -752,12 +648,12 @@ filter_unlink (call_frame_t *frame,
 		if (loc->inode->st_mode & S_IWOTH)
 			break;
 		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
+		STACK_UNWIND (frame, -1, EPERM, NULL, NULL);
 		return 0;
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS);
+		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
 		return 0;
 	}
 	STACK_WIND (frame,
@@ -773,9 +669,16 @@ filter_rmdir_cbk (call_frame_t *frame,
 		  void *cookie,
 		  xlator_t *this,
 		  int32_t op_ret,
-		  int32_t op_errno)
+		  int32_t op_errno,
+                  struct iatt *preparent,
+                  struct iatt *postparent)
 {
-	STACK_UNWIND (frame, op_ret, op_errno);
+        if (op_ret >= 0) {
+		update_stat (preparent, this->private);
+		update_stat (postparent, this->private);
+        }
+
+	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
 	return 0;
 }
 
@@ -801,13 +704,13 @@ filter_rmdir (call_frame_t *frame,
 		if (loc->inode->st_mode & S_IWOTH)
 			break;
 		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
+		STACK_UNWIND (frame, -1, EPERM, NULL, NULL);
 		return 0;
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-			STACK_UNWIND (frame, -1, EROFS);
-			return 0;
+                STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
+                return 0;
 	}			
 	STACK_WIND (frame,
 		    filter_rmdir_cbk,
@@ -824,18 +727,24 @@ filter_symlink_cbk (call_frame_t *frame,
 		    int32_t op_ret,
 		    int32_t op_errno,
 		    inode_t *inode,
-		    struct stat *buf)
+                    struct iatt *buf,
+                    struct iatt *preparent,
+                    struct iatt *postparent)
 {
 	int ret = 0;
 	if (op_ret >= 0) {
 		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->st_uid);
+		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
 		if (ret == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"couldn't set context");
 		}
+
+		update_stat (preparent, this->private);
+		update_stat (postparent, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
+	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
+                      preparent, postparent);
 	return 0;
 }
 
@@ -856,13 +765,15 @@ filter_symlink (call_frame_t *frame,
 		if (parent->st_mode & S_IWOTH)
 			break;
 		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
+		STACK_UNWIND (frame, -1, EPERM, NULL, NULL,
+                              NULL, NULL);
 		return 0;
 
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
+		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
+                              NULL, NULL);
 		return 0;
 	}
 	STACK_WIND (frame,
@@ -880,12 +791,25 @@ filter_rename_cbk (call_frame_t *frame,
 		   xlator_t *this,
 		   int32_t op_ret,
 		   int32_t op_errno,
-		   struct stat *buf)
+		   struct iatt *buf,
+                   struct iatt *preoldparent,
+                   struct iatt *postoldparent,
+                   struct iatt *prenewparent,
+                   struct iatt *postnewparent)
 {
 	if (op_ret >= 0) {
 		update_stat (buf, this->private);
+
+		update_stat (preoldparent, this->private);
+		update_stat (postoldparent, this->private);
+
+		update_stat (prenewparent, this->private);
+		update_stat (postnewparent, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
+
+	STACK_UNWIND (frame, op_ret, op_errno, buf,
+                      preoldparent, postoldparent,
+                      prenewparent, postnewparent);
 	return 0;
 }
 
@@ -913,13 +837,17 @@ filter_rename (call_frame_t *frame,
 			break;
 		gf_log (this->name, GF_LOG_DEBUG, 
 			"%s -> %s: returning permission denied", oldloc->path, newloc->path);
-		STACK_UNWIND (frame, -1, EPERM);
+		STACK_UNWIND (frame, -1, EPERM, NULL,
+                              NULL, NULL,
+                              NULL, NULL);
 		return 0;
 
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL);
+		STACK_UNWIND (frame, -1, EROFS, NULL,
+                              NULL, NULL,
+                              NULL, NULL);
 		return 0;
 	}
 	STACK_WIND (frame,
@@ -938,18 +866,24 @@ filter_link_cbk (call_frame_t *frame,
 		 int32_t op_ret,
 		 int32_t op_errno,
 		 inode_t *inode,
-		 struct stat *buf)
+                 struct iatt *buf,
+                 struct iatt *preparent,
+                 struct iatt *postparent)
 {
 	int ret = 0;
 	if (op_ret >= 0) {
 		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->st_uid);
+		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
 		if (ret == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"couldn't set context");
 		}
+
+		update_stat (preparent, this->private);
+		update_stat (postparent, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode,	buf);
+	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
+                      preparent, postparent);
 	return 0;
 }
 
@@ -965,7 +899,8 @@ filter_link (call_frame_t *frame,
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
+		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
+                              NULL, NULL);
 		return 0;
 	}
 	STACK_WIND (frame,
@@ -985,18 +920,23 @@ filter_create_cbk (call_frame_t *frame,
 		   int32_t op_errno,
 		   fd_t *fd,
 		   inode_t *inode,
-		   struct stat *buf)
+		   struct iatt *buf,
+                   struct iatt *preparent,
+                   struct iatt *postparent)
 {
 	int ret = 0;
 	if (op_ret >= 0) {
 		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->st_uid);
+		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
 		if (ret == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"couldn't set context");
 		}
+		update_stat (preparent, this->private);
+		update_stat (postparent, this->private);
 	}
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
+	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf,
+                      preparent, postparent);
 	return 0;
 }
 
@@ -1018,13 +958,15 @@ filter_create (call_frame_t *frame,
 		if (parent->st_mode & S_IWOTH)
 			break;
 		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
+		STACK_UNWIND (frame, -1, EPERM, NULL, NULL, NULL,
+                              NULL, NULL);
 		return 0;
 
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL, NULL);
+		STACK_UNWIND (frame, -1, EROFS, NULL, NULL, NULL,
+                              NULL, NULL);
 		return 0;
 	}
 	STACK_WIND (frame, filter_create_cbk,
@@ -1051,7 +993,8 @@ filter_open (call_frame_t *frame,
 	     xlator_t *this,
 	     loc_t *loc,
 	     int32_t flags, 
-	     fd_t *fd)
+	     fd_t *fd,
+             int32_t wbflags)
 {
 	int32_t ret = 0;
 	ret = update_frame (frame, loc->inode, this->private);
@@ -1059,13 +1002,15 @@ filter_open (call_frame_t *frame,
 	case GF_FILTER_MAP_UID:
 		if (loc->inode->st_mode & S_IWGRP)
 			break;
-		if (!((flags & O_WRONLY) || (flags & O_RDWR)) 
+		if (!(((flags & O_ACCMODE) == O_WRONLY)
+                      || ((flags & O_ACCMODE) == O_RDWR))
 		    && (loc->inode->st_mode & S_IRGRP))
 			break;
 	case GF_FILTER_MAP_BOTH:
 		if (loc->inode->st_mode & S_IWOTH)
 				break;
-		if (!((flags & O_WRONLY) || (flags & O_RDWR))
+		if (!(((flags & O_ACCMODE) == O_WRONLY)
+                      || ((flags & O_ACCMODE) == O_RDWR))
 		    && (loc->inode->st_mode & S_IROTH))
 			break;
 		gf_log (this->name, GF_LOG_DEBUG, 
@@ -1076,7 +1021,8 @@ filter_open (call_frame_t *frame,
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		if (!((flags & O_WRONLY) || (flags & O_RDWR)))
+		if (!(((flags & O_ACCMODE) == O_WRONLY)
+                      || ((flags & O_ACCMODE) == O_RDWR)))
 			break;
 		STACK_UNWIND (frame, -1, EROFS, NULL);
 		return 0;
@@ -1086,7 +1032,7 @@ filter_open (call_frame_t *frame,
 		    filter_open_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd);
+		    loc, flags, fd, wbflags);
 	return 0;
 }
 
@@ -1098,7 +1044,8 @@ filter_readv_cbk (call_frame_t *frame,
 		  int32_t op_errno,
 		  struct iovec *vector,
 		  int32_t count,
-		  struct stat *stbuf)
+		  struct iatt *stbuf,
+                  struct iobref *iobref)
 {
 	if (op_ret >= 0) {
 		update_stat (stbuf, this->private);
@@ -1108,7 +1055,8 @@ filter_readv_cbk (call_frame_t *frame,
 		      op_errno,
 		      vector,
 		      count,
-		      stbuf);
+		      stbuf,
+                      iobref);
 	return 0;
 }
 
@@ -1136,15 +1084,18 @@ filter_writev_cbk (call_frame_t *frame,
 		   xlator_t *this,
 		   int32_t op_ret,
 		   int32_t op_errno,
-		   struct stat *stbuf)
+                   struct iatt *prebuf,
+		   struct iatt *postbuf)
 {
 	if (op_ret >= 0) {
-		update_stat (stbuf, this->private);
+		update_stat (prebuf, this->private);
+		update_stat (postbuf, this->private);
 	}
 	STACK_UNWIND (frame,
 		      op_ret,
 		      op_errno,
-		      stbuf);
+		      prebuf,
+                      postbuf);
 	return 0;
 }
 
@@ -1154,7 +1105,8 @@ filter_writev (call_frame_t *frame,
 	       fd_t *fd,
 	       struct iovec *vector,
 	       int32_t count,
-	       off_t off)
+	       off_t off,
+               struct iobref *iobref)
 {
 	int32_t ret = 0;
 	ret = update_frame (frame, fd->inode, this->private);
@@ -1162,7 +1114,7 @@ filter_writev (call_frame_t *frame,
 	case GF_FILTER_FILTER_UID:
 	case GF_FILTER_FILTER_GID:
 	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL);
+		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
 		return 0;
 	}
 
@@ -1173,7 +1125,8 @@ filter_writev (call_frame_t *frame,
 		    fd,
 		    vector,
 		    count,
-		    off);
+		    off,
+                    iobref);
 	return 0;
 }
 
@@ -1183,7 +1136,7 @@ filter_fstat_cbk (call_frame_t *frame,
 		  xlator_t *this,
 		  int32_t op_ret,
 		  int32_t op_errno,
-		  struct stat *buf)
+		  struct iatt *buf)
 {
 	if (op_ret >= 0) {
 		update_stat (buf, this->private);
@@ -1393,6 +1346,25 @@ filter_removexattr (call_frame_t *frame,
 	return 0;
 }
 
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_filter_mt_end + 1);
+        
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
 int32_t 
 init (xlator_t *this)
 {
@@ -1422,7 +1394,7 @@ init (xlator_t *this)
 			"dangling volume. check volfile ");
 	}
 
-	filter = CALLOC (sizeof (*filter), 1);
+	filter = GF_CALLOC (sizeof (*filter), 1, gf_filter_mt_gf_filter);
 	ERR_ABORT (filter);
 	
 	if (dict_get (this->options, "read-only")) {
@@ -1457,11 +1429,11 @@ init (xlator_t *this)
 		option_data = dict_get (this->options, "translate-uid");
 		value = strtok_r (option_data->data, ",", &tmp_str);
 		while (value) {
-			dup_str = strdup (value);
+			dup_str = gf_strdup (value);
 			input_value_str1 = strtok_r (dup_str, "=", &tmp_str1);
 			if (input_value_str1) {
 				/* Check for n-m */
-				char *temp_string = strdup (input_value_str1);
+				char *temp_string = gf_strdup (input_value_str1);
 				input_value_str2 = strtok_r (temp_string, "-", &tmp_str2);
 				if (gf_string2int (input_value_str2, &input_value) != 0) {
 					gf_log (this->name, GF_LOG_ERROR, 
@@ -1480,7 +1452,7 @@ init (xlator_t *this)
 					}
 				}
 				filter->translate_input_uid[filter->translate_num_uid_entries][1] = input_value;
-				FREE (temp_string);
+				GF_FREE (temp_string);
 				output_value_str = strtok_r (NULL, "=", &tmp_str1);
 				if (output_value_str) {
 					if (gf_string2int (output_value_str, &output_value) != 0) {
@@ -1509,7 +1481,7 @@ init (xlator_t *this)
 			if (filter->translate_num_uid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
 				break;
 			value = strtok_r (NULL, ",", &tmp_str);
-			FREE (dup_str);
+			GF_FREE (dup_str);
 		}
 	}
 
@@ -1521,11 +1493,11 @@ init (xlator_t *this)
 		option_data = dict_get (this->options, "translate-gid");
 		value = strtok_r (option_data->data, ",", &tmp_str);
 		while (value) {
-			dup_str = strdup (value);
+			dup_str = gf_strdup (value);
 			input_value_str1 = strtok_r (dup_str, "=", &tmp_str1);
 			if (input_value_str1) {
 				/* Check for n-m */
-				char *temp_string = strdup (input_value_str1);
+				char *temp_string = gf_strdup (input_value_str1);
 				input_value_str2 = strtok_r (temp_string, "-", &tmp_str2);
 				if (gf_string2int (input_value_str2, &input_value) != 0) {
 					gf_log (this->name, GF_LOG_ERROR, 
@@ -1544,7 +1516,7 @@ init (xlator_t *this)
 					}
 				}
 				filter->translate_input_gid[filter->translate_num_gid_entries][1] = input_value;
-				FREE (temp_string);
+				GF_FREE (temp_string);
 				output_value_str = strtok_r (NULL, "=", &tmp_str1);
 				if (output_value_str) {
 					if (gf_string2int (output_value_str, &output_value) != 0) {
@@ -1574,7 +1546,7 @@ init (xlator_t *this)
 			if (filter->translate_num_gid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
 				break;
 			value = strtok_r (NULL, ",", &tmp_str);
-			FREE (dup_str);
+			GF_FREE (dup_str);
 		}
 	}
 
@@ -1585,7 +1557,7 @@ init (xlator_t *this)
 		option_data = dict_get (this->options, "filter-uid");
 		value = strtok_r (option_data->data, ",", &tmp_str);
 		while (value) {
-			dup_str = strdup (value);
+			dup_str = gf_strdup (value);
 			/* Check for n-m */
 			input_value_str1 = strtok_r (dup_str, "-", &tmp_str1);
 			if (gf_string2int (input_value_str1, &input_value) != 0) {
@@ -1615,7 +1587,7 @@ init (xlator_t *this)
 			if (filter->filter_num_uid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
 				break;
 			value = strtok_r (NULL, ",", &tmp_str);
-			FREE (dup_str);
+			GF_FREE (dup_str);
 		}
 		filter->partial_filter = 1;
 	}
@@ -1627,7 +1599,7 @@ init (xlator_t *this)
 		option_data = dict_get (this->options, "filter-gid");
 		value = strtok_r (option_data->data, ",", &tmp_str);
 		while (value) {
-			dup_str = strdup (value);
+			dup_str = gf_strdup (value);
 			/* Check for n-m */
 			input_value_str1 = strtok_r (dup_str, "-", &tmp_str1);
 			if (gf_string2int (input_value_str1, &input_value) != 0) {
@@ -1657,7 +1629,7 @@ init (xlator_t *this)
 			if (filter->filter_num_gid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
 				break;
 			value = strtok_r (NULL, ",", &tmp_str);
-			FREE (dup_str);
+			GF_FREE (dup_str);
 		}
 		gf_log (this->name, GF_LOG_ERROR, "this option is not supported currently.. exiting");
 		return -1;
@@ -1698,7 +1670,7 @@ fini (xlator_t *this)
 {
 	struct gf_filter *filter = this->private;
 
-	FREE (filter);
+	GF_FREE (filter);
 
 	return;
 }
@@ -1708,8 +1680,6 @@ struct xlator_fops fops = {
 	.lookup      = filter_lookup,
 	.stat        = filter_stat,
 	.fstat       = filter_fstat,
-	.chmod       = filter_chmod,
-	.fchmod      = filter_fchmod,
 	.readlink    = filter_readlink,
 	.mknod       = filter_mknod,
 	.mkdir       = filter_mkdir,
@@ -1718,8 +1688,6 @@ struct xlator_fops fops = {
 	.symlink     = filter_symlink,
 	.rename      = filter_rename,
 	.link        = filter_link,
-	.chown       = filter_chown,
-	.fchown      = filter_fchown,
 	.truncate    = filter_truncate,
 	.ftruncate   = filter_ftruncate,
 	.create      = filter_create,
@@ -1730,10 +1698,8 @@ struct xlator_fops fops = {
 	.getxattr    = filter_getxattr,
 	.removexattr = filter_removexattr,
 	.opendir     = filter_opendir,
-	.utimens     = filter_utimens,
-};
-
-struct xlator_mops mops = {
+        .setattr     = filter_setattr,
+        .fsetattr    = filter_fsetattr,
 };
 
 struct xlator_cbks cbks = {
diff --git a/xlators/performance/stat-prefetch/Makefile.am b/xlators/features/gfid-access/Makefile.am
index af437a64d..af437a64d 100644
--- a/xlators/performance/stat-prefetch/Makefile.am
+++ b/xlators/features/gfid-access/Makefile.am
diff --git a/xlators/features/gfid-access/src/Makefile.am b/xlators/features/gfid-access/src/Makefile.am
new file mode 100644
index 000000000..db53affaa
--- /dev/null
+++ b/xlators/features/gfid-access/src/Makefile.am
@@ -0,0 +1,15 @@
+xlator_LTLIBRARIES = gfid-access.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+gfid_access_la_LDFLAGS = -module -avoid-version
+
+gfid_access_la_SOURCES = gfid-access.c
+gfid_access_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = gfid-access.h gfid-access-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/gfid-access/src/gfid-access-mem-types.h b/xlators/features/gfid-access/src/gfid-access-mem-types.h
new file mode 100644
index 000000000..168d67b43
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access-mem-types.h
@@ -0,0 +1,23 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GFID_ACCESS_MEM_TYPES_H
+#define _GFID_ACCESS_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_changelog_mem_types {
+        gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
+        gf_gfid_access_mt_gfid_t,
+        gf_gfid_access_mt_end
+};
+
+#endif
+
diff --git a/xlators/features/gfid-access/src/gfid-access.c b/xlators/features/gfid-access/src/gfid-access.c
new file mode 100644
index 000000000..da0ba7e50
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access.c
@@ -0,0 +1,1172 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "gfid-access.h"
+#include "inode.h"
+#include "byte-order.h"
+
+
+
+void
+ga_newfile_args_free (ga_newfile_args_t *args)
+{
+        if (!args)
+                goto out;
+
+        GF_FREE (args->bname);
+
+        if (S_ISLNK (args->st_mode) && args->args.symlink.linkpath) {
+                GF_FREE (args->args.symlink.linkpath);
+                args->args.symlink.linkpath = NULL;
+        }
+
+        mem_put (args);
+out:
+        return;
+}
+
+
+void
+ga_heal_args_free (ga_heal_args_t *args)
+{
+        if (!args)
+                goto out;
+
+        GF_FREE (args->bname);
+
+        mem_put (args);
+out:
+        return;
+}
+
+
+ga_newfile_args_t *
+ga_newfile_parse_args (xlator_t *this, data_t *data)
+{
+        ga_newfile_args_t *args     = NULL;
+        ga_private_t      *priv     = NULL;
+        int                len      = 0;
+        int                blob_len = 0;
+        int                min_len  = 0;
+        void              *blob     = NULL;
+
+        priv = this->private;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        min_len = sizeof (args->uid) + sizeof (args->gid) + sizeof (args->gfid)
+                + sizeof (args->st_mode) + 2 + 2;
+        if (blob_len < min_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid length: Total length is less "
+                        "than minimum length.");
+                goto err;
+        }
+
+        args = mem_get0 (priv->newfile_args_pool);
+        if (args == NULL)
+                goto err;
+
+        args->uid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        args->gid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        args->st_mode = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+        if (len == blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. No null byte present.",
+                        args->gfid);
+                goto err;
+        }
+
+        args->bname = GF_CALLOC (1, (len + 1), gf_common_mt_char);
+        if (args->bname == NULL)
+                goto err;
+
+        memcpy (args->bname, blob, (len + 1));
+        blob += (len + 1);
+        blob_len -= (len + 1);
+
+        if (S_ISDIR (args->st_mode)) {
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mkdir.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mkdir.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+                if (blob_len < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+        } else if (S_ISLNK (args->st_mode)) {
+                len = strnlen (blob, blob_len);
+                if (len == blob_len) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.symlink.linkpath = GF_CALLOC (1, len + 1,
+                                                         gf_common_mt_char);
+                if (args->args.symlink.linkpath == NULL)
+                        goto err;
+
+                memcpy (args->args.symlink.linkpath, blob, (len + 1));
+                blob += (len + 1);
+                blob_len -= (len + 1);
+        } else {
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.rdev = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+        }
+
+        if (blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. Invalid length",
+                        args->gfid);
+                goto err;
+        }
+
+        return args;
+
+err:
+        if (args)
+                ga_newfile_args_free (args);
+
+        return NULL;
+}
+
+ga_heal_args_t *
+ga_heal_parse_args (xlator_t *this, data_t *data)
+{
+        ga_heal_args_t *args     = NULL;
+        ga_private_t   *priv     = NULL;
+        void           *blob     = NULL;
+        int             len      = 0;
+        int             blob_len = 0;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        priv = this->private;
+
+        /* bname should at least contain a character */
+        if (blob_len < (sizeof (args->gfid) + 2))
+                goto err;
+
+        args = mem_get0 (priv->heal_args_pool);
+        if (!args)
+                goto err;
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+                goto err;
+
+        args->bname = GF_CALLOC (1, len + 1, gf_common_mt_char);
+        if (!args->bname)
+                goto err;
+
+        memcpy (args->bname, blob, len);
+        blob_len -= (len + 1);
+
+        if (blob_len)
+                goto err;
+
+        return args;
+
+err:
+        if (args)
+                ga_heal_args_free (args);
+
+        return NULL;
+}
+
+static int32_t
+ga_fill_tmp_loc (loc_t *loc, xlator_t *this, char *gfid,
+                 char *bname, dict_t *xdata, loc_t *new_loc)
+{
+        int       ret    = -1;
+        uint64_t  value  = 0;
+        inode_t  *parent = NULL;
+
+        parent = loc->inode;
+        ret = inode_ctx_get (loc->inode, this, &value);
+        if (!ret) {
+                parent = (void *)value;
+        }
+
+        /* parent itself should be looked up */
+        uuid_copy (new_loc->pargfid, parent->gfid);
+        new_loc->parent = inode_ref (parent);
+
+        new_loc->inode = inode_grep (parent->table, parent, bname);
+        if (!new_loc->inode)
+                new_loc->inode = inode_new (parent->table);
+
+        loc_path (new_loc, bname);
+        new_loc->name = basename (new_loc->path);
+
+        /* As GFID would not be set on the entry yet, lets not send entry
+           gfid in the request */
+        /*uuid_copy (new_loc->gfid, (const unsigned char *)gfid); */
+
+        ret = dict_set_static_bin (xdata, "gfid-req", gfid, 16);
+        if (ret < 0)
+                goto out;
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+
+static gf_boolean_t
+__is_gfid_access_dir (uuid_t gfid)
+{
+        uuid_t  aux_gfid;
+
+        memset (aux_gfid, 0, 16);
+        aux_gfid[15] = GF_AUX_GFID;
+
+        if (uuid_compare (gfid, aux_gfid) == 0)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+int32_t
+ga_forget (xlator_t *this, inode_t *inode)
+{
+        int       ret = -1;
+        uint64_t  value = 0;
+        inode_t  *tmp_inode = NULL;
+
+        ret = inode_ctx_del (inode, this, &value);
+        if (ret)
+                goto out;
+
+        tmp_inode = (void *)value;
+        inode_unref (tmp_inode);
+
+out:
+        return 0;
+}
+
+
+static int
+ga_heal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno,
+             inode_t *inode, struct iatt *stat, dict_t *dict,
+             struct iatt *postparent)
+{
+        call_frame_t *orig_frame = NULL;
+
+        orig_frame = frame->local;
+        frame->local = NULL;
+
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, orig_frame, op_ret, op_errno, dict);
+
+        return 0;
+}
+
+static int
+ga_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t *xdata)
+{
+        call_frame_t *orig_frame = NULL;
+
+        orig_frame = frame->local;
+        frame->local = NULL;
+
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, orig_frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_new_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+              dict_t *xdata)
+{
+        int                ret       = -1;
+        ga_newfile_args_t *args      = NULL;
+        loc_t              tmp_loc   = {0,};
+        call_frame_t      *new_frame = NULL;
+        mode_t             mode      = 0;
+
+        args = ga_newfile_parse_args (this, data);
+        if (!args)
+                goto out;
+
+        if (!xdata)
+                xdata = dict_new ();
+
+        ret = ga_fill_tmp_loc (loc, this, args->gfid,
+                               args->bname, xdata, &tmp_loc);
+        if (ret)
+                goto out;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
+        new_frame->local = (void *)frame;
+
+        new_frame->root->uid = args->uid;
+        new_frame->root->gid = args->gid;
+
+        if (S_ISDIR (args->st_mode)) {
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                            &tmp_loc, args->args.mkdir.mode,
+                            args->args.mkdir.umask, xdata);
+        } else if (S_ISLNK (args->st_mode)) {
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                            args->args.symlink.linkpath,
+                            &tmp_loc, 0, xdata);
+        } else {
+                /* use 07777 (4 7s) for considering the Sticky bits etc) */
+                mode = (S_IFMT & args->st_mode) |
+                        (07777 | args->args.mknod.mode);;
+
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                            &tmp_loc, mode,
+                            args->args.mknod.rdev, args->args.mknod.umask,
+                            xdata);
+        }
+
+        ret = 0;
+out:
+        ga_newfile_args_free (args);
+
+        return ret;
+}
+
+int32_t
+ga_heal_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+               dict_t *xdata)
+{
+        int             ret       = -1;
+        ga_heal_args_t *args      = NULL;
+        loc_t           tmp_loc   = {0,};
+        call_frame_t   *new_frame = NULL;
+
+        args = ga_heal_parse_args (this, data);
+        if (!args)
+                goto out;
+
+        if (!xdata)
+                xdata = dict_new ();
+
+        ret = ga_fill_tmp_loc (loc, this, args->gfid, args->bname,
+                               xdata, &tmp_loc);
+        if (ret)
+                goto out;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
+        new_frame->local = (void *)frame;
+
+        STACK_WIND (new_frame, ga_heal_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    &tmp_loc, xdata);
+
+        ret = 0;
+out:
+        if (args)
+                ga_heal_args_free (args);
+
+        return ret;
+}
+
+int32_t
+ga_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+ga_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+
+        data_t  *data     = NULL;
+        int      op_errno = ENOMEM;
+        int      ret      = 0;
+        inode_t *unref    = NULL;
+
+        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
+            ((loc->parent &&
+              __is_root_gfid (loc->parent->gfid)) ||
+             __is_root_gfid (loc->pargfid))) {
+                op_errno = EPERM;
+                goto err;
+        }
+
+        data = dict_get (dict, GF_FUSE_AUX_GFID_NEWFILE);
+        if (data) {
+                ret = ga_new_entry (frame, this, loc, data, xdata);
+                if (ret)
+                        goto err;
+                return 0;
+        }
+
+        data = dict_get (dict, GF_FUSE_AUX_GFID_HEAL);
+        if (data) {
+                ret = ga_heal_entry (frame, this, loc, data, xdata);
+                if (ret)
+                        goto err;
+                return 0;
+        }
+
+        //If the inode is a virtual inode change the inode otherwise perform
+        //the operation on same inode
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, ga_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+        return 0;
+}
+
+
+int32_t
+ga_virtual_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        int       j           = 0;
+        int       i           = 0;
+        int       ret         = 0;
+        uint64_t  temp_ino    = 0;
+        inode_t  *cbk_inode   = NULL;
+        inode_t  *true_inode  = NULL;
+        uuid_t    random_gfid = {0,};
+
+        if (frame->local)
+                cbk_inode = frame->local;
+        else
+                cbk_inode = inode;
+
+        frame->local = NULL;
+        if (op_ret)
+                goto unwind;
+
+        if (!IA_ISDIR (buf->ia_type))
+                goto unwind;
+
+        /* need to send back a different inode for linking in itable */
+        if (cbk_inode == inode) {
+                /* check if the inode is in the 'itable' or
+                   if its just previously discover()'d inode */
+                true_inode = inode_find (inode->table, buf->ia_gfid);
+                if (!true_inode) {
+                        cbk_inode = inode_new (inode->table);
+
+                        if (!cbk_inode) {
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+                                goto unwind;
+                        }
+                        /* the inode is not present in itable, ie, the actual
+                           path is not yet looked up. Use the current inode
+                           itself for now */
+                        inode_ref (inode);
+                } else {
+                        /* 'inode_ref()' has been done in inode_find() */
+                        inode = true_inode;
+                }
+
+                ret = inode_ctx_put (cbk_inode, this, (uint64_t)inode);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set the inode ctx with"
+                                "the actual inode");
+                        if (inode)
+                                inode_unref (inode);
+                }
+                inode = NULL;
+        }
+
+        if (!uuid_is_null (cbk_inode->gfid)) {
+                /* if the previous linked inode is used, use the
+                   same gfid */
+                uuid_copy (random_gfid, cbk_inode->gfid);
+        } else {
+                /* replace the buf->ia_gfid to a random gfid
+                   for directory, for files, what we received is fine */
+                uuid_generate (random_gfid);
+        }
+
+        uuid_copy (buf->ia_gfid, random_gfid);
+
+        for (i = 15; i > (15 - 8); i--) {
+                temp_ino += (uint64_t)(buf->ia_gfid[i]) << j;
+                j += 8;
+        }
+        buf->ia_ino = temp_ino;
+
+unwind:
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, cbk_inode, buf,
+                             xdata, postparent);
+
+        return 0;
+}
+
+int32_t
+ga_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        ga_private_t *priv = NULL;
+
+        /* if the entry in question is not 'root',
+           then follow the normal path */
+        if (op_ret || !__is_root_gfid(buf->ia_gfid))
+                goto unwind;
+
+        priv = this->private;
+
+        /* do we need to copy root stbuf everytime? */
+        /* mostly yes, as we want to have the 'stat' info show latest
+           in every _cbk() */
+
+        /* keep the reference for root stat buf */
+        priv->root_stbuf = *buf;
+        priv->gfiddir_stbuf = priv->root_stbuf;
+        priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
+        priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
+
+unwind:
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
+        return 0;
+}
+
+int32_t
+ga_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        ga_private_t *priv     = NULL;
+        int           ret      = -1;
+        uuid_t        tmp_gfid = {0,};
+        loc_t         tmp_loc  = {0,};
+        uint64_t      value    = 0;
+        inode_t      *inode    = NULL;
+        inode_t      *true_inode    = NULL;
+        int32_t       op_errno = ENOENT;
+
+        /* if its discover(), no need for any action here */
+        if (!loc->name)
+                goto wind;
+
+        /* if its revalidate, and inode is not of type directory,
+           proceed with 'wind' */
+        if (loc->inode && loc->inode->ia_type &&
+            !IA_ISDIR (loc->inode->ia_type))
+                goto wind;
+
+        priv = this->private;
+
+        /* need to check if the lookup is on virtual dir */
+        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
+            ((loc->parent && __is_root_gfid (loc->parent->gfid)) ||
+             __is_root_gfid (loc->pargfid))) {
+                /* this means, the query is on '/.gfid', return the fake stat,
+                   and say success */
+
+                STACK_UNWIND_STRICT (lookup, frame, 0, 0, loc->inode,
+                                     &priv->gfiddir_stbuf, xdata,
+                                     &priv->root_stbuf);
+                return 0;
+        }
+
+        /* now, check if the lookup() is on an existing entry,
+           but on gfid-path */
+        if (!((loc->parent && __is_gfid_access_dir (loc->parent->gfid)) ||
+              __is_gfid_access_dir (loc->pargfid)))
+                goto wind;
+
+        /* make sure the 'basename' is actually a 'canonical-gfid',
+           otherwise, return error */
+        ret = uuid_parse (loc->name, tmp_gfid);
+        if (ret)
+                goto err;
+
+        /* if its fresh lookup, go ahead and send it down, if not,
+           for directory, we need indirection to actual dir inode */
+        if (!(loc->inode && loc->inode->ia_type))
+                goto discover;
+
+        /* revalidate on directory */
+        ret = inode_ctx_get (loc->inode, this, &value);
+        if (ret)
+                goto err;
+
+        inode = (void *)value;
+
+        /* valid inode, already looked up, work on that */
+        if (inode->ia_type)
+                goto discover;
+
+        /* check if the inode is in the 'itable' or
+           if its just previously discover()'d inode */
+        true_inode = inode_find (loc->inode->table, tmp_gfid);
+        if (true_inode) {
+                /* time do another lookup and update the context
+                   with proper inode */
+                op_errno = ESTALE;
+                goto err;
+        }
+
+discover:
+        /* for the virtual entries, we don't need to send 'gfid-req' key, as
+           for these entries, we don't want to 'set' a new gfid */
+        if (xdata)
+                dict_del (xdata, "gfid-req");
+
+        uuid_copy (tmp_loc.gfid, tmp_gfid);
+
+        /* if revalidate, then we need to have the proper reference */
+        if (inode) {
+                tmp_loc.inode = inode_ref (inode);
+                frame->local = loc->inode;
+        } else {
+                tmp_loc.inode = inode_ref (loc->inode);
+        }
+
+        STACK_WIND (frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+
+        inode_unref (tmp_loc.inode);
+
+        return 0;
+
+wind:
+        /* used for all the normal lookup path */
+        STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, loc->inode,
+                             &priv->gfiddir_stbuf, xdata,
+                             &priv->root_stbuf);
+        return 0;
+}
+
+int
+ga_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                    xdata);
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, loc->inode,
+                             NULL, NULL, NULL, xdata);
+        return 0;
+}
+
+
+int
+ga_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_create_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, xdata);
+
+        return 0;
+
+}
+
+int
+ga_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_symlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkname, loc, umask, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev,
+                    umask, xdata);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+          dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_rmdir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flag, xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL,
+                             NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+           dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_unlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL,
+                             NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_rename (call_frame_t *frame, xlator_t *this,
+           loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *oldloc_unref = NULL;
+        inode_t *newloc_unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
+        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, oldloc, oldloc_unref,
+                                         handle_newloc);
+
+handle_newloc:
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, newloc, newloc_unref, wind);
+
+wind:
+        STACK_WIND (frame, default_rename_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+
+        if (oldloc_unref)
+                inode_unref (oldloc_unref);
+
+        if (newloc_unref)
+                inode_unref (newloc_unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+
+int
+ga_link (call_frame_t *frame, xlator_t *this,
+         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *oldloc_unref = NULL;
+        inode_t *newloc_unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
+        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, oldloc, oldloc_unref,
+                                         handle_newloc);
+
+handle_newloc:
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, newloc, newloc_unref, wind);
+
+wind:
+        STACK_WIND (frame, default_link_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+
+        if (oldloc_unref)
+                inode_unref (oldloc_unref);
+
+        if (newloc_unref)
+                inode_unref (newloc_unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            fd_t *fd, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        /* also check if the loc->inode itself is virtual
+           inode, if yes, return with failure, mainly because we
+           can't handle all the readdirp and other things on it. */
+        if (inode_ctx_get (loc->inode, this, NULL) == 0) {
+                op_errno = ENOTSUP;
+                goto err;
+        }
+
+        STACK_WIND (frame, default_opendir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             const char *name, dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
+         dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct iatt *stbuf, int32_t valid,
+            dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_gfid_access_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
+                        " init failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        ga_private_t *priv = NULL;
+        int ret = -1;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "not configured with exactly one child. exiting");
+                goto out;
+        }
+
+        /* This can be the top of graph in certain cases */
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dangling volume. check volfile ");
+        }
+
+        /* TODO: define a mem-type structure */
+        priv = GF_CALLOC (1, sizeof (*priv), gf_gfid_access_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        priv->newfile_args_pool = mem_pool_new (ga_newfile_args_t, 512);
+        if (!priv->newfile_args_pool)
+                goto out;
+
+        priv->heal_args_pool = mem_pool_new (ga_heal_args_t, 512);
+        if (!priv->heal_args_pool)
+                goto out;
+
+        this->private = priv;
+
+        ret = 0;
+out:
+        if (ret && priv) {
+                if (priv->newfile_args_pool)
+                        mem_pool_destroy (priv->newfile_args_pool);
+                GF_FREE (priv);
+        }
+
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        ga_private_t *priv = NULL;
+        priv = this->private;
+        this->private = NULL;
+
+        if (priv) {
+                if (priv->newfile_args_pool)
+                        mem_pool_destroy (priv->newfile_args_pool);
+                if (priv->heal_args_pool)
+                        mem_pool_destroy (priv->heal_args_pool);
+                GF_FREE (priv);
+        }
+
+        return;
+}
+
+
+struct xlator_fops fops = {
+        .lookup = ga_lookup,
+
+        /* entry fops */
+        .mkdir   = ga_mkdir,
+        .mknod   = ga_mknod,
+        .create  = ga_create,
+        .symlink = ga_symlink,
+        .link    = ga_link,
+        .unlink  = ga_unlink,
+        .rmdir   = ga_rmdir,
+        .rename  = ga_rename,
+
+        /* handle any other directory operations here */
+        .opendir  = ga_opendir,
+        .stat     = ga_stat,
+        .setattr  = ga_setattr,
+        .getxattr = ga_getxattr,
+        .removexattr = ga_removexattr,
+
+        /* special fop to handle more entry creations */
+        .setxattr = ga_setxattr,
+};
+
+struct xlator_cbks cbks = {
+        .forget = ga_forget,
+};
+
+struct volume_options options[] = {
+        /* This translator doesn't take any options, or provide any options */
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/gfid-access/src/gfid-access.h b/xlators/features/gfid-access/src/gfid-access.h
new file mode 100644
index 000000000..e13c9b724
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access.h
@@ -0,0 +1,128 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __GFID_ACCESS_H__
+#define __GFID_ACCESS_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "defaults.h"
+#include "gfid-access-mem-types.h"
+
+#define UUID_CANONICAL_FORM_LEN 36
+
+#define GF_FUSE_AUX_GFID_NEWFILE "glusterfs.gfid.newfile"
+#define GF_FUSE_AUX_GFID_HEAL    "glusterfs.gfid.heal"
+
+#define GF_GFID_KEY "GLUSTERFS_GFID"
+#define GF_GFID_DIR ".gfid"
+#define GF_AUX_GFID 0xd
+
+#define GFID_ACCESS_GET_VALID_DIR_INODE(x,l,unref,lbl) do {             \
+                int       ret       = 0;                                \
+                uint64_t  value     = 0;                                \
+                inode_t  *tmp_inode = NULL;                             \
+                                                                        \
+                /* if its an entry operation, on the virtual */         \
+                /* directory inode as parent, we need to handle */      \
+                /* it properly */                                       \
+                if (l->parent) {                                        \
+                        ret = inode_ctx_get (l->parent, x, &value);     \
+                        if (ret)                                        \
+                                goto lbl;                               \
+                        tmp_inode = (inode_t *)value;                   \
+                        unref = inode_ref (tmp_inode);                  \
+                        l->parent = tmp_inode;                          \
+                        /* if parent is virtual, no need to handle */   \
+                        /* loc->inode */                                \
+                        break;                                          \
+                }                                                       \
+                                                                        \
+                /* if its an inode operation, on the virtual */         \
+                /* directory inode itself, we need to handle */         \
+                /* it properly */                                       \
+                if (l->inode) {                                         \
+                        ret = inode_ctx_get (l->inode, x, &value);      \
+                        if (ret)                                        \
+                                goto lbl;                               \
+                        tmp_inode = (inode_t *)value;                   \
+                        unref = inode_ref (tmp_inode);                  \
+                        l->inode = tmp_inode;                           \
+                }                                                       \
+                                                                        \
+        } while (0)
+
+#define GFID_ACCESS_ENTRY_OP_CHECK(loc,err,lbl)    do {                 \
+                /* need to check if the lookup is on virtual dir */     \
+                if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&  \
+                    ((loc->parent &&                                    \
+                      __is_root_gfid (loc->parent->gfid)) ||            \
+                      __is_root_gfid (loc->pargfid))) {                 \
+                        err = EEXIST;                                   \
+                        goto lbl;                                       \
+                }                                                       \
+                                                                        \
+                /* now, check if the lookup() is on an existing */      \
+                /* entry, but on gfid-path */                           \
+                if ((loc->parent &&                                     \
+                     __is_gfid_access_dir (loc->parent->gfid)) ||       \
+                    __is_gfid_access_dir (loc->pargfid)) {              \
+                        err = EPERM;                                    \
+                        goto lbl;                                       \
+                }                                                       \
+        } while (0)
+
+
+typedef struct {
+        unsigned int  uid;
+        unsigned int  gid;
+        char          gfid[UUID_CANONICAL_FORM_LEN + 1];
+        unsigned int  st_mode;
+        char         *bname;
+
+        union {
+                struct _symlink_in {
+                        char     *linkpath;
+                } __attribute__ ((__packed__)) symlink;
+
+                struct _mknod_in {
+                        unsigned int mode;
+                        unsigned int rdev;
+                        unsigned int umask;
+                } __attribute__ ((__packed__)) mknod;
+
+                struct _mkdir_in {
+                        unsigned int mode;
+                        unsigned int umask;
+                } __attribute__ ((__packed__)) mkdir;
+        } __attribute__ ((__packed__)) args;
+} __attribute__((__packed__)) ga_newfile_args_t;
+
+typedef struct {
+        char      gfid[UUID_CANONICAL_FORM_LEN + 1];
+        char     *bname; /* a null terminated basename */
+} __attribute__((__packed__)) ga_heal_args_t;
+
+struct ga_private {
+        /* root inode's stbuf */
+        struct iatt root_stbuf;
+        struct iatt gfiddir_stbuf;
+        struct mem_pool *newfile_args_pool;
+        struct mem_pool *heal_args_pool;
+};
+typedef struct ga_private ga_private_t;
+
+#endif /* __GFID_ACCESS_H__ */
diff --git a/xlators/features/glupy/Makefile.am b/xlators/features/glupy/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/glupy/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/glupy/doc/README.md b/xlators/features/glupy/doc/README.md
new file mode 100644
index 000000000..2d7b30ef6
--- /dev/null
+++ b/xlators/features/glupy/doc/README.md
@@ -0,0 +1,44 @@
+This is just the very start for a GlusterFS[1] meta-translator that will
+allow translator code to be written in Python.  It's based on the standard
+Python embedding (not extending) techniques, plus a dash of the ctypes module.
+The interface is a pretty minimal adaptation of the dispatches and callbacks
+from the C API[2] to Python, as follows:
+
+* Dispatch functions and callbacks must be defined on an "xlator" class
+  derived from gluster.Translator so that they'll be auto-registered with
+  the C translator during initialization.
+
+* For each dispatch or callback function you want to intercept, you define a
+  Python function using the xxx\_fop\_t or xxx\_cbk\_t decorator.
+
+* The arguments for each operation are different, so you'll need to refer to
+  the C API.  GlusterFS-specific types are used (though only loc\_t is fully
+  defined so far) and type correctness is enforced by ctypes.
+
+* If you do intercept a dispatch function, it is your responsibility to call
+  xxx\_wind (like STACK\_WIND in the C API but operation-specific) to pass
+  the request to the next translator.  If you do not intercept a function, it
+  will default the same way as for C (pass through to the same operation with
+  the same arguments on the first child translator).
+
+* If you intercept a callback function, it is your responsibility to call
+  xxx\_unwind (like STACK\_UNWIND\_STRICT in the C API) to pass the request back
+  to the caller.
+
+So far only the lookup and create operations are handled this way, to support
+the "negative lookup" example.  Now that the basic infrastructure is in place,
+adding more functions should be very quick, though with that much boilerplate I
+might pause to write a code generator.  I also plan to add structure
+definitions and interfaces for some of the utility functions in libglusterfs
+(especially those having to do with inode and fd context) in the fairly near
+future.  Note that you can also use ctypes to get at anything not explicitly
+exposed to Python already.
+
+_If you're coming here because of the Linux Journal article, please note that
+the code has evolved since that was written. The version that matches the
+article is here:_
+
+https://github.com/jdarcy/glupy/tree/4bbae91ba459ea46ef32f2966562492e4ca9187a
+
+[1] http://www.gluster.org
+[2] http://hekafs.org/dist/xlator_api_2.html
diff --git a/xlators/features/glupy/doc/TESTING b/xlators/features/glupy/doc/TESTING
new file mode 100644
index 000000000..e05f17f49
--- /dev/null
+++ b/xlators/features/glupy/doc/TESTING
@@ -0,0 +1,9 @@
+Loading a translator written in Python using the glupy meta translator
+-------------------------------------------------------------------------------
+'test.vol' is a simple volfile with the debug-trace Python translator on top
+of a brick. The volfile can be mounted using the following command.
+
+$ glusterfs --debug -f test.vol /path/to/mntpt
+
+If then file operations are performed on the newly mounted file system, log
+output would be printed by the Python translator on the standard output.
diff --git a/xlators/features/glupy/doc/test.vol b/xlators/features/glupy/doc/test.vol
new file mode 100644
index 000000000..0751a488c
--- /dev/null
+++ b/xlators/features/glupy/doc/test.vol
@@ -0,0 +1,10 @@
+volume vol-posix
+    type storage/posix
+    option directory /path/to/brick
+end-volume
+
+volume vol-glupy
+    type features/glupy
+    option module-name debug-trace
+    subvolumes vol-posix
+end-volume
diff --git a/xlators/features/glupy/src/Makefile.am b/xlators/features/glupy/src/Makefile.am
new file mode 100644
index 000000000..960862839
--- /dev/null
+++ b/xlators/features/glupy/src/Makefile.am
@@ -0,0 +1,20 @@
+xlator_LTLIBRARIES = glupy.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+glupydir = $(xlatordir)/glupy
+
+glupy_PYTHON = gluster.py negative.py helloworld.py debug-trace.py
+
+glupy_la_LDFLAGS = -module -avoid-version -shared -nostartfiles
+glupy_la_SOURCES = glupy.c
+glupy_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	 -lpthread -l$(BUILD_PYTHON_LIB)
+
+noinst_HEADERS = glupy.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -isystem $(BUILD_PYTHON_INC)
+
+AM_CFLAGS = -Wall -fno-strict-aliasing -DGLUSTER_PYTHON_PATH=\"$(glupydir)\" $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/glupy/src/debug-trace.py b/xlators/features/glupy/src/debug-trace.py
new file mode 100644
index 000000000..53e76546b
--- /dev/null
+++ b/xlators/features/glupy/src/debug-trace.py
@@ -0,0 +1,774 @@
+import sys
+import stat
+from uuid import UUID
+from time import strftime, localtime
+from gluster import *
+# This translator was written primarily to test the fop entry point definitions
+# and structure definitions in 'gluster.py'.
+# It is similar to the debug-trace translator, one of the already available
+# translator types written in C, that logs the arguments passed to the fops and
+# their corresponding cbk functions.
+
+dl.get_id.restype = c_long
+dl.get_id.argtypes = [ POINTER(call_frame_t) ]
+
+dl.get_rootunique.restype = c_uint64
+dl.get_rootunique.argtypes = [ POINTER(call_frame_t) ]
+
+def uuid2str (gfid):
+        return str(UUID(''.join(map("{0:02x}".format, gfid))))
+
+
+def st_mode_from_ia (prot, filetype):
+        st_mode = 0
+        type_bit = 0
+        prot_bit = 0
+
+        if filetype == IA_IFREG:
+                type_bit = stat.S_IFREG
+        elif filetype == IA_IFDIR:
+                type_bit = stat.S_IFDIR
+        elif filetype == IA_IFLNK:
+                type_bit = stat.S_IFLNK
+        elif filetype == IA_IFBLK:
+                type_bit = stat.S_IFBLK
+        elif filetype == IA_IFCHR:
+                type_bit = stat.S_IFCHR
+        elif filetype == IA_IFIFO:
+                type_bit = stat.S_IFIFO
+        elif filetype == IA_IFSOCK:
+                type_bit = stat.S_IFSOCK
+        elif filetype == IA_INVAL:
+                pass
+
+
+        if prot.suid:
+                prot_bit |= stat.S_ISUID
+        if prot.sgid:
+                prot_bit |= stat.S_ISGID
+        if prot.sticky:
+                prot_bit |= stat.S_ISVTX
+
+        if prot.owner.read:
+                prot_bit |= stat.S_IRUSR
+        if prot.owner.write:
+                prot_bit |= stat.S_IWUSR
+        if prot.owner.execn:
+                prot_bit |= stat.S_IXUSR
+
+        if prot.group.read:
+                prot_bit |= stat.S_IRGRP
+        if prot.group.write:
+                prot_bit |= stat.S_IWGRP
+        if prot.group.execn:
+                prot_bit |= stat.S_IXGRP
+
+        if prot.other.read:
+                prot_bit |= stat.S_IROTH
+        if prot.other.write:
+                prot_bit |= stat.S_IWOTH
+        if prot.other.execn:
+                prot_bit |= stat.S_IXOTH
+
+        st_mode = (type_bit | prot_bit)
+
+        return st_mode
+
+
+def trace_stat2str (buf):
+        gfid = uuid2str(buf.contents.ia_gfid)
+        mode = st_mode_from_ia(buf.contents.ia_prot, buf.contents.ia_type)
+        atime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_atime))
+        mtime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_mtime))
+        ctime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_ctime))
+        return ("(gfid={0:s}, ino={1:d}, mode={2:o}, nlink={3:d}, uid ={4:d}, "+
+                "gid ={5:d}, size={6:d}, blocks={7:d}, atime={8:s}, mtime={9:s}, "+
+                "ctime={10:s})").format(gfid, buf.contents.ia_no, mode,
+                                        buf.contents.ia_nlink,
+                                        buf.contents.ia_uid,
+                                        buf.contents.ia_gid,
+                                        buf.contents.ia_size,
+                                        buf.contents.ia_blocks,
+                                        atime_buf, mtime_buf,
+                                        ctime_buf)
+
+class xlator(Translator):
+
+        def __init__(self, c_this):
+                Translator.__init__(self, c_this)
+                self.gfids = {}
+
+        def lookup_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.gfid)
+                print("GLUPY TRACE LOOKUP FOP- {0:d}: gfid={1:s}; " +
+                      "path={2:s}").format(unique, gfid, loc.contents.path)
+                self.gfids[key] = gfid
+                dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def lookup_cbk(self, frame, cookie, this, op_ret, op_errno,
+                       inode, buf, xdata, postparent):
+                unique =dl.get_rootunique(frame)
+                key =dl.get_id(frame)
+                if op_ret == 0:
+                        gfid = uuid2str(buf.contents.ia_gfid)
+                        statstr = trace_stat2str(buf)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE LOOKUP CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *buf={3:s}; " +
+                              "*postparent={4:s}").format(unique, gfid,
+                                                          op_ret, statstr,
+                                                          postparentstr)
+                else:
+                        gfid = self.gfids[key]
+                        print("GLUPY TRACE LOOKUP CBK - {0:d}: gfid={1:s};" +
+                              " op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                      gfid,
+                                                                      op_ret,
+                                                                      op_errno)
+                del self.gfids[key]
+                dl.unwind_lookup(frame, cookie, this, op_ret, op_errno,
+                                 inode, buf, xdata, postparent)
+                return 0
+
+        def create_fop(self, frame, this, loc, flags, mode, umask, fd,
+                       xdata):
+                unique = dl.get_rootunique(frame)
+                gfid = uuid2str(loc.contents.gfid)
+                print("GLUPY TRACE CREATE FOP- {0:d}: gfid={1:s}; path={2:s}; " +
+                      "fd={3:s}; flags=0{4:o}; mode=0{5:o}; " +
+                      "umask=0{6:o}").format(unique, gfid, loc.contents.path,
+                                             fd, flags, mode, umask)
+                dl.wind_create(frame, POINTER(xlator_t)(), loc, flags,mode,
+                               umask, fd, xdata)
+                return 0
+
+        def create_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
+                       inode, buf, preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret >= 0:
+                        gfid = uuid2str(inode.contents.gfid)
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE CREATE CBK- {0:d}: gfid={1:s};" +
+                              " op_ret={2:d}; fd={3:s}; *stbuf={4:s}; " +
+                              "*preparent={5:s};" +
+                              " *postparent={6:s}").format(unique, gfid, op_ret,
+                                                           fd, statstr,
+                                                           preparentstr,
+                                                           postparentstr)
+                else:
+                        print ("GLUPY TRACE CREATE CBK- {0:d}: op_ret={1:d}; " +
+                              "op_errno={2:d}").format(unique, op_ret, op_errno)
+                dl.unwind_create(frame, cookie, this, op_ret, op_errno, fd,
+                                 inode, buf, preparent, postparent, xdata)
+                return 0
+
+        def open_fop(self, frame, this, loc, flags, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE OPEN FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "flags={3:d}; fd={4:s}").format(unique, gfid,
+                                                      loc.contents.path, flags,
+                                                      fd)
+                self.gfids[key] = gfid
+                dl.wind_open(frame, POINTER(xlator_t)(), loc, flags, fd, xdata)
+                return 0
+
+        def open_cbk(self, frame, cookie, this, op_ret, op_errno, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE OPEN CBK- {0:d}: gfid={1:s}; op_ret={2:d}; "
+                      "op_errno={3:d}; *fd={4:s}").format(unique, gfid,
+                                                          op_ret, op_errno, fd)
+                del self.gfids[key]
+                dl.unwind_open(frame, cookie, this, op_ret, op_errno, fd,
+                               xdata)
+                return 0
+
+        def readv_fop(self, frame, this, fd, size, offset, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READV FOP- {0:d}: gfid={1:s}; "+
+                      "fd={2:s}; size ={3:d}; offset={4:d}; " +
+                      "flags=0{5:x}").format(unique, gfid, fd, size, offset,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_readv (frame, POINTER(xlator_t)(), fd, size, offset,
+                               flags, xdata)
+                return 0
+
+        def readv_cbk(self, frame, cookie, this, op_ret, op_errno, vector,
+                      count, buf, iobref, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret >= 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
+                              "op_ret={2:d}; *buf={3:s};").format(unique, gfid,
+                                                                  op_ret,
+                                                                  statstr)
+
+                else:
+                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_readv (frame, cookie, this, op_ret, op_errno,
+                                 vector, count, buf, iobref, xdata)
+                return 0
+
+        def writev_fop(self, frame, this, fd, vector, count, offset, flags,
+                       iobref, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE  WRITEV FOP- {0:d}: gfid={1:s}; " +
+                      "fd={2:s}; count={3:d}; offset={4:d}; " +
+                      "flags=0{5:x}").format(unique, gfid, fd, count, offset,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_writev(frame, POINTER(xlator_t)(), fd, vector, count,
+                               offset, flags, iobref, xdata)
+                return 0
+
+        def writev_cbk(self, frame, cookie, this, op_ret, op_errno, prebuf,
+                       postbuf, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                if op_ret >= 0:
+                        preopstr = trace_stat2str(prebuf)
+                        postopstr = trace_stat2str(postbuf)
+                        print("GLUPY TRACE WRITEV CBK- {0:d}: op_ret={1:d}; " +
+                              "*prebuf={2:s}; " +
+                              "*postbuf={3:s}").format(unique, op_ret, preopstr,
+                                                       postopstr)
+                else:
+                        gfid = self.gfids[key]
+                        print("GLUPY TRACE WRITEV CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_writev (frame, cookie, this, op_ret, op_errno,
+                                  prebuf, postbuf, xdata)
+                return 0
+
+        def opendir_fop(self, frame, this, loc, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE OPENDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "fd={3:s}").format(unique, gfid, loc.contents.path, fd)
+                self.gfids[key] = gfid
+                dl.wind_opendir(frame, POINTER(xlator_t)(), loc, fd, xdata)
+                return 0
+
+        def opendir_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
+                        xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE OPENDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
+                      " op_errno={3:d}; fd={4:s}").format(unique, gfid, op_ret,
+                                                          op_errno, fd)
+                del self.gfids[key]
+                dl.unwind_opendir(frame, cookie, this, op_ret, op_errno,
+                                  fd, xdata)
+                return 0
+
+        def readdir_fop(self, frame, this, fd, size, offset, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READDIR FOP- {0:d}:  gfid={1:s}; fd={2:s}; " +
+                      "size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
+                                                         offset)
+                self.gfids[key] = gfid
+                dl.wind_readdir(frame, POINTER(xlator_t)(), fd, size, offset,
+                                xdata)
+                return 0
+
+        def readdir_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                        xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE READDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
+                      " op_errno={3:d}").format(unique, gfid, op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_readdir(frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata)
+                return 0
+
+        def readdirp_fop(self, frame, this, fd, size, offset, dictionary):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READDIRP FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
+                      " size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
+                                                          offset)
+                self.gfids[key] = gfid
+                dl.wind_readdirp(frame, POINTER(xlator_t)(), fd, size, offset,
+                                 dictionary)
+                return 0
+
+        def readdirp_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                         xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE READDIRP CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_readdirp(frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata)
+                return 0
+
+        def mkdir_fop(self, frame, this, loc, mode, umask, xdata):
+                unique = dl.get_rootunique(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE MKDIR FOP- {0:d}: gfid={1:s}; path={2:s}; " +
+                      "mode={3:d}; umask=0{4:o}").format(unique, gfid,
+                                                         loc.contents.path, mode,
+                                                         umask)
+                dl.wind_mkdir(frame, POINTER(xlator_t)(), loc, mode, umask,
+                              xdata)
+                return 0
+
+        def mkdir_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                      preparent, postparent,  xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret == 0:
+                        gfid = uuid2str(inode.contents.gfid)
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE MKDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *stbuf={3:s}; *prebuf={4:s}; "+
+                              "*postbuf={5:s} ").format(unique, gfid, op_ret,
+                                                        statstr,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE MKDIR CBK- {0:d}:  op_ret={1:d}; "+
+                              "op_errno={2:d}").format(unique, op_ret, op_errno)
+                dl.unwind_mkdir(frame, cookie, this, op_ret, op_errno, inode,
+                                buf, preparent, postparent, xdata)
+                return 0
+
+        def rmdir_fop(self, frame, this, loc, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE RMDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "flags={3:d}").format(unique, gfid, loc.contents.path,
+                                            flags)
+                self.gfids[key] = gfid
+                dl.wind_rmdir(frame, POINTER(xlator_t)(), loc, flags, xdata)
+                return 0
+
+        def rmdir_cbk(self, frame, cookie, this, op_ret, op_errno, preparent,
+                      postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *prebuf={3:s}; "+
+                              "*postbuf={4:s}").format(unique, gfid, op_ret,
+                                                       preparentstr,
+                                                       postparentstr)
+                else:
+                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_rmdir(frame, cookie, this, op_ret, op_errno,
+                                preparent, postparent, xdata)
+                return 0
+
+        def stat_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE STAT FOP- {0:d}: gfid={1:s}; " +
+                      " path={2:s}").format(unique, gfid, loc.contents.path)
+                self.gfids[key] = gfid
+                dl.wind_stat(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def stat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                     xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d};  *buf={3:s};").format(unique,
+                                                                   gfid,
+                                                                   op_ret,
+                                                                   statstr)
+                else:
+                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_stat(frame, cookie, this, op_ret, op_errno,
+                               buf, xdata)
+                return 0
+
+        def fstat_fop(self, frame, this, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FSTAT FOP- {0:d}:  gfid={1:s}; " +
+                      "fd={2:s}").format(unique, gfid, fd)
+                self.gfids[key] = gfid
+                dl.wind_fstat(frame, POINTER(xlator_t)(), fd, xdata)
+                return 0
+
+        def fstat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                      xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; *buf={3:s}").format(unique,
+                                                                  gfid,
+                                                                  op_ret,
+                                                                  statstr)
+                else:
+                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique.
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_fstat(frame, cookie, this, op_ret, op_errno,
+                                buf, xdata)
+                return 0
+
+        def statfs_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                if loc.contents.inode:
+                        gfid = uuid2str(loc.contents.inode.contents.gfid)
+                else:
+                        gfid = "0"
+                print("GLUPY TRACE STATFS FOP- {0:d}: gfid={1:s}; "+
+                      "path={2:s}").format(unique, gfid, loc.contents.path)
+                dl.wind_statfs(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def statfs_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                       xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret == 0:
+                        #TBD: print buf (pointer to an iovec type object)
+                        print("GLUPY TRACE STATFS CBK {0:d}: "+
+                              "op_ret={1:d}").format(unique, op_ret)
+                else:
+                        print("GLUPY TRACE STATFS CBK-  {0:d}"+
+                              "op_ret={1:d}; op_errno={2:d}").format(unique,
+                                                                     op_ret,
+                                                                     op_errno)
+                dl.unwind_statfs(frame, cookie, this, op_ret, op_errno,
+                                 buf, xdata)
+                return 0
+
+        def getxattr_fop(self, frame, this, loc, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE GETXATTR FOP- {0:d}: gfid={1:s}; path={2:s};"+
+                      " name={3:s}").format(unique, gfid, loc.contents.path,
+                                            name)
+                self.gfids[key]=gfid
+                dl.wind_getxattr(frame, POINTER(xlator_t)(), loc, name, xdata)
+                return 0
+
+        def getxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                         dictionary, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE GETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}; "+
+                      " dictionary={4:s}").format(unique, gfid, op_ret, op_errno,
+                                                  dictionary)
+                del self.gfids[key]
+                dl.unwind_getxattr(frame, cookie, this, op_ret, op_errno,
+                                   dictionary, xdata)
+                return 0
+
+        def fgetxattr_fop(self, frame, this, fd, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FGETXATTR FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
+                      "name={3:s}").format(unique, gfid, fd, name)
+                self.gfids[key] = gfid
+                dl.wind_fgetxattr(frame, POINTER(xlator_t)(), fd, name, xdata)
+                return 0
+
+        def fgetxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                          dictionary, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE FGETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d};"+
+                      " dictionary={4:s}").format(unique, gfid, op_ret,
+                                                  op_errno, dictionary)
+                del self.gfids[key]
+                dl.unwind_fgetxattr(frame, cookie, this, op_ret, op_errno,
+                                    dictionary, xdata)
+                return 0
+
+        def setxattr_fop(self, frame, this, loc, dictionary, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE SETXATTR FOP- {0:d}:  gfid={1:s}; path={2:s};"+
+                      " flags={3:d}").format(unique, gfid, loc.contents.path,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_setxattr(frame, POINTER(xlator_t)(), loc, dictionary,
+                                 flags, xdata)
+                return 0
+
+        def setxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE SETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_setxattr(frame, cookie, this, op_ret, op_errno,
+                                   xdata)
+                return 0
+
+        def fsetxattr_fop(self, frame, this, fd, dictionary, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FSETXATTR FOP- {0:d}: gfid={1:s}; fd={2:p}; "+
+                      "flags={3:d}").format(unique, gfid, fd, flags)
+                self.gfids[key] = gfid
+                dl.wind_fsetxattr(frame, POINTER(xlator_t)(), fd, dictionary,
+                                  flags, xdata)
+                return 0
+
+        def fsetxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE FSETXATTR CBK- {0:d}: gfid={1:s};  "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_fsetxattr(frame, cookie, this, op_ret, op_errno,
+                                   xdata)
+                return 0
+
+        def removexattr_fop(self, frame, this, loc, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE REMOVEXATTR FOP- {0:d}:  gfid={1:s}; "+
+                      "path={2:s}; name={3:s}").format(unique, gfid,
+                                                       loc.contents.path,
+                                                       name)
+                self.gfids[key] = gfid
+                dl.wind_removexattr(frame, POINTER(xlator_t)(), loc, name,
+                                    xdata)
+                return 0
+
+        def removexattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                            xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE REMOVEXATTR CBK- {0:d}: gfid={1:s} "+
+                      " op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                              op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_removexattr(frame, cookie, this, op_ret, op_errno,
+                                      xdata)
+                return 0
+
+        def link_fop(self, frame, this, oldloc, newloc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                if (newloc.contents.inode):
+                        newgfid = uuid2str(newloc.contents.inode.contents.gfid)
+                else:
+                        newgfid = "0"
+                oldgfid = uuid2str(oldloc.contents.inode.contents.gfid)
+                print("GLUPY TRACE LINK FOP-{0:d}: oldgfid={1:s}; oldpath={2:s};"+
+                      "newgfid={3:s};"+
+                      "newpath={4:s}").format(unique, oldgfid,
+                                              oldloc.contents.path,
+                                              newgfid,
+                                              newloc.contents.path)
+                self.gfids[key] =  oldgfid
+                dl.wind_link(frame, POINTER(xlator_t)(), oldloc, newloc,
+                             xdata)
+                return 0
+
+        def link_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                     preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE LINK CBK- {0:d}: op_ret={1:d} "+
+                              "*stbuf={2:s}; *prebuf={3:s}; "+
+                              "*postbuf={4:s} ").format(unique, op_ret, statstr,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE LINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; "+
+                              "op_errno={3:d}").format(unique, gfid,
+                                                       op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_link(frame, cookie, this, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata)
+                return 0
+
+        def unlink_fop(self, frame, this, loc, xflag, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE UNLINK FOP- {0:d}; gfid={1:s}; path={2:s}; "+
+                      "flag={3:d}").format(unique, gfid, loc.contents.path,
+                                           xflag)
+                self.gfids[key] = gfid
+                dl.wind_unlink(frame, POINTER(xlator_t)(), loc, xflag,
+                               xdata)
+                return 0
+
+        def unlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                       preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE UNLINK CBK- {0:d}: gfid ={1:s}; "+
+                              "op_ret={2:d}; *prebuf={3:s}; "+
+                              "*postbuf={4:s} ").format(unique, gfid, op_ret,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE UNLINK CBK: {0:d}: gfid ={1:s}; "+
+                              "op_ret={2:d}; "+
+                              "op_errno={3:d}").format(unique, gfid, op_ret,
+                                                       op_errno)
+                del self.gfids[key]
+                dl.unwind_unlink(frame, cookie, this, op_ret, op_errno,
+                                 preparent, postparent, xdata)
+                return 0
+
+        def readlink_fop(self, frame, this, loc, size, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE READLINK FOP- {0:d}:  gfid={1:s}; path={2:s};"+
+                      " size={3:d}").format(unique, gfid, loc.contents.path,
+                                            size)
+                self.gfids[key] = gfid
+                dl.wind_readlink(frame, POINTER(xlator_t)(), loc, size,
+                               xdata)
+                return 0
+
+        def readlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                         buf, stbuf, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid  = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(stbuf)
+                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; op_errno={3:d}; *prebuf={4:s}; "+
+                              "*postbuf={5:s} ").format(unique, gfid,
+                                                        op_ret, op_errno,
+                                                        buf, statstr)
+                else:
+                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                      gfid,
+                                                                      op_ret,
+                                                                      op_errno)
+                del self.gfids[key]
+                dl.unwind_readlink(frame, cookie, this, op_ret, op_errno, buf,
+                                   stbuf, xdata)
+                return 0
+
+        def symlink_fop(self, frame, this, linkpath, loc, umask, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE SYMLINK FOP- {0:d}: gfid={1:s}; "+
+                      "linkpath={2:s}; path={3:s};"+
+                      "umask=0{4:o}").format(unique, gfid, linkpath,
+                                             loc.contents.path, umask)
+                self.gfids[key] = gfid
+                dl.wind_symlink(frame, POINTER(xlator_t)(), linkpath, loc,
+                                umask, xdata)
+                return 0
+
+        def symlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                        inode, buf, preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *stbuf={3:s}; *preparent={4:s}; "+
+                              "*postparent={5:s}").format(unique, gfid,
+                                                          op_ret, statstr,
+                                                          preparentstr,
+                                                          postparentstr)
+                else:
+                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_symlink(frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata)
+                return 0
diff --git a/xlators/features/glupy/src/glupy.c b/xlators/features/glupy/src/glupy.c
new file mode 100644
index 000000000..dc86c0071
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.c
@@ -0,0 +1,2470 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  GlusterFS is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published
+  by the Free Software Foundation; either version 3 of the License,
+  or (at your option) any later version.
+
+  GlusterFS is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see
+  <http://www.gnu.org/licenses/>.
+*/
+
+#include <ctype.h>
+#include <sys/uio.h>
+#include <Python.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "glupy.h"
+
+/* UTILITY FUNCTIONS FOR FOP-SPECIFIC CODE */
+
+pthread_key_t gil_init_key;
+
+PyGILState_STATE
+glupy_enter (void)
+{
+#if 0
+        if (!pthread_getspecific(gil_init_key)) {
+                PyEval_ReleaseLock();
+                (void)pthread_setspecific(gil_init_key,(void *)1);
+        }
+#endif
+
+        return PyGILState_Ensure();
+}
+
+void
+glupy_leave (PyGILState_STATE gstate)
+{
+        PyGILState_Release(gstate);
+}
+
+/* FOP: LOOKUP */
+
+int32_t
+glupy_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_LOOKUP]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_lookup_cbk_t)(priv->cbks[GLUPY_LOOKUP]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, xdata, postparent);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
+        return 0;
+}
+
+int32_t
+glupy_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_LOOKUP]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_lookup_t)(priv->fops[GLUPY_LOOKUP]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+        return 0;
+}
+
+void
+wind_lookup (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_lookup_cbk,xl,xl->fops->lookup,loc,xdata);
+}
+
+void
+unwind_lookup (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(lookup,frame,op_ret,op_errno,
+                            inode,buf,xdata,postparent);
+}
+
+void
+set_lookup_fop (long py_this, fop_lookup_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_LOOKUP] = (long)fop;
+}
+
+void
+set_lookup_cbk (long py_this, fop_lookup_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_LOOKUP] = (long)cbk;
+}
+
+/* FOP: CREATE */
+
+int32_t
+glupy_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_CREATE]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_create_cbk_t)(priv->cbks[GLUPY_CREATE]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_CREATE]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_create_t)(priv->fops[GLUPY_CREATE]))(
+                frame, this, loc, flags, mode, umask, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_create_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
+        return 0;
+}
+
+void
+wind_create (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_create_cbk,xl, xl->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+}
+
+void
+unwind_create (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_create_fop (long py_this, fop_create_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_CREATE] = (long)fop;
+}
+
+void
+set_create_cbk (long py_this, fop_create_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_CREATE] = (long)cbk;
+}
+
+/* FOP: OPEN */
+
+int32_t
+glupy_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_OPEN]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_open_cbk_t)(priv->cbks[GLUPY_OPEN]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int32_t
+glupy_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            int32_t flags, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_OPEN]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_open_t)(priv->fops[GLUPY_OPEN]))(
+                frame, this, loc, flags, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_open_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+        return 0;
+}
+
+void
+wind_open (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_open_cbk, xl, xl->fops->open, loc, flags,
+                    fd, xdata);
+}
+
+void
+unwind_open (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+}
+
+void
+set_open_fop (long py_this, fop_open_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_OPEN] = (long)fop;
+}
+
+void
+set_open_cbk (long py_this, fop_open_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_OPEN] = (long)cbk;
+}
+
+/* FOP: READV */
+
+int32_t
+glupy_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                 int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                 dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READV]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readv_cbk_t)(priv->cbks[GLUPY_READV]))(
+                frame, cookie, this, op_ret, op_errno,
+                vector, count, stbuf, iobref, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, stbuf, iobref, xdata);
+        return 0;
+}
+
+int32_t
+glupy_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READV]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readv_t)(priv->fops[GLUPY_READV]))(
+                frame, this, fd, size, offset, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_readv (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_readv_cbk, xl, xl->fops->readv, fd, size,
+                    offset, flags, xdata);
+}
+
+void
+unwind_readv (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct iovec *vector,
+              int32_t count, struct iatt *stbuf, struct iobref *iobref,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, stbuf, iobref, xdata);
+}
+
+void
+set_readv_fop (long py_this, fop_readv_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_READV] = (long)fop;
+}
+
+void
+set_readv_cbk (long py_this, fop_readv_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_READV] = (long)cbk;
+}
+
+/* FOP: WRITEV */
+
+int32_t
+glupy_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_WRITEV]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_writev_cbk_t)(priv->cbks[GLUPY_WRITEV]))(
+                frame, cookie, this, op_ret, op_errno,
+                prebuf, postbuf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int32_t count, off_t offset,
+              uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_WRITEV]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_writev_t)(priv->fops[GLUPY_WRITEV]))(
+                frame, this, fd, vector, count, offset, flags,
+                iobref, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_writev_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
+                    offset, flags, iobref, xdata);
+        return 0;
+}
+
+void
+wind_writev (call_frame_t *frame, xlator_t *xl, fd_t *fd, struct iovec *vector,
+             int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+             dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_writev_cbk, xl, xl->fops->writev, fd, vector,
+                    count, offset, flags, iobref, xdata);
+}
+
+void
+unwind_writev (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+}
+
+void
+set_writev_fop (long py_this, fop_writev_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_WRITEV] = (long)fop;
+}
+
+void
+set_writev_cbk (long py_this, fop_writev_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_WRITEV] = (long)cbk;
+}
+
+
+/* FOP: OPENDIR */
+
+int32_t
+glupy_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd,
+                   dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_OPENDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_opendir_cbk_t)(priv->cbks[GLUPY_OPENDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int32_t
+glupy_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_OPENDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_opendir_t)(priv->fops[GLUPY_OPENDIR]))(
+                frame, this, loc, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_opendir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+        return 0;
+}
+
+void
+wind_opendir (call_frame_t *frame, xlator_t *xl, loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_opendir_cbk,xl,xl->fops->opendir,loc,fd,xdata);
+}
+
+void
+unwind_opendir (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(opendir,frame,op_ret,op_errno,
+                            fd,xdata);
+}
+
+void
+set_opendir_fop (long py_this, fop_opendir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_OPENDIR] = (long)fop;
+}
+
+void
+set_opendir_cbk (long py_this, fop_opendir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_OPENDIR] = (long)cbk;
+}
+
+/* FOP: READDIR */
+
+int32_t
+glupy_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                   dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readdir_cbk_t)(priv->cbks[GLUPY_READDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                entries,  xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               size_t size, off_t offset, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readdir_t)(priv->fops[GLUPY_READDIR]))(
+                frame, this, fd, size, offset, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir,fd, size, offset, xdata);
+        return 0;
+}
+
+void
+wind_readdir(call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+             off_t offset, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_readdir_cbk,xl,xl->fops->readdir,fd,size,offset,xdata);
+}
+
+void
+unwind_readdir (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(readdir,frame,op_ret,op_errno,
+                            entries, xdata);
+}
+
+void
+set_readdir_fop (long py_this, fop_readdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READDIR] = (long)fop;
+}
+
+void
+set_readdir_cbk (long py_this, fop_readdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READDIR] = (long)cbk;
+}
+
+
+/* FOP: READDIRP */
+
+int32_t
+glupy_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READDIRP]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readdirp_cbk_t)(priv->cbks[GLUPY_READDIRP]))(
+                frame, cookie, this, op_ret, op_errno,
+                entries,  xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                size_t size, off_t offset, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READDIRP]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readdirp_t)(priv->fops[GLUPY_READDIRP]))(
+                frame, this, fd, size, offset, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readdirp_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdirp,fd, size, offset, xdata);
+        return 0;
+}
+
+void
+wind_readdirp (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+               off_t offset, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_readdirp_cbk,xl,xl->fops->readdirp,fd,size,offset,xdata);
+}
+
+void
+unwind_readdirp (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                 dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(readdirp,frame,op_ret,op_errno,
+                            entries, xdata);
+}
+
+void
+set_readdirp_fop (long py_this, fop_readdirp_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READDIRP] = (long)fop;
+}
+
+void
+set_readdirp_cbk (long py_this, fop_readdirp_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READDIRP] = (long)cbk;
+}
+
+
+/* FOP:STAT */
+
+int32_t
+glupy_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_STAT]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_stat_cbk_t)(priv->cbks[GLUPY_STAT]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_STAT]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_stat_t)(priv->fops[GLUPY_STAT]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+}
+
+void
+wind_stat (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_stat_cbk,xl,xl->fops->stat,loc,xdata);
+}
+
+void
+unwind_stat (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, struct iatt *buf,
+             dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(stat,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_stat_fop (long py_this, fop_stat_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_STAT] = (long)fop;
+}
+
+void
+set_stat_cbk (long py_this, fop_stat_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_STAT] = (long)cbk;
+}
+
+
+/* FOP: FSTAT */
+
+int32_t
+glupy_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FSTAT]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fstat_cbk_t)(priv->cbks[GLUPY_FSTAT]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FSTAT]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fstat_t)(priv->fops[GLUPY_FSTAT]))(
+                frame, this, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fstat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+}
+
+void
+wind_fstat (call_frame_t *frame, xlator_t *xl, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_fstat_cbk,xl,xl->fops->fstat,fd,xdata);
+}
+
+void
+unwind_fstat (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct iatt *buf,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(fstat,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_fstat_fop (long py_this, fop_fstat_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FSTAT] = (long)fop;
+}
+
+void
+set_fstat_cbk (long py_this, fop_fstat_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FSTAT] = (long)cbk;
+}
+
+/* FOP:STATFS */
+
+int32_t
+glupy_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_STATFS]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_statfs_cbk_t)(priv->cbks[GLUPY_STATFS]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_STATFS]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_statfs_t)(priv->fops[GLUPY_STATFS]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_statfs_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->statfs, loc, xdata);
+        return 0;
+}
+
+void
+wind_statfs (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_statfs_cbk,xl,xl->fops->statfs,loc,xdata);
+}
+
+void
+unwind_statfs (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+               dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(statfs,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_statfs_fop (long py_this, fop_statfs_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_STATFS] = (long)fop;
+}
+
+void
+set_statfs_cbk (long py_this, fop_statfs_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_STATFS] = (long)cbk;
+}
+
+
+/* FOP: SETXATTR */
+
+int32_t
+glupy_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_SETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_setxattr_cbk_t)(priv->cbks[GLUPY_SETXATTR]))(
+                frame, cookie, this, op_ret, op_errno,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_SETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_setxattr_t)(priv->fops[GLUPY_SETXATTR]))(
+                frame, this, loc, dict, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_setxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_setxattr_cbk, xl, xl->fops->setxattr,
+                    loc, dict, flags, xdata);
+}
+
+
+void
+unwind_setxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_setxattr_fop (long py_this, fop_setxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_SETXATTR] = (long)fop;
+}
+
+void
+set_setxattr_cbk (long py_this, fop_setxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_SETXATTR] = (long)cbk;
+}
+
+/* FOP: GETXATTR */
+
+int32_t
+glupy_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_GETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_getxattr_cbk_t)(priv->cbks[GLUPY_GETXATTR]))(
+                frame, cookie, this, op_ret, op_errno, dict,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_GETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_getxattr_t)(priv->fops[GLUPY_GETXATTR]))(
+                frame, this, loc, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_getxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_getxattr_cbk, xl, xl->fops->getxattr,
+                    loc, name, xdata);
+}
+
+
+void
+unwind_getxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict,
+                 dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+
+}
+
+
+void
+set_getxattr_fop (long py_this, fop_getxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_GETXATTR] = (long)fop;
+}
+
+
+void
+set_getxattr_cbk (long py_this, fop_getxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_GETXATTR] = (long)cbk;
+}
+
+/* FOP: FSETXATTR */
+
+int32_t
+glupy_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FSETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fsetxattr_cbk_t)(priv->cbks[GLUPY_FSETXATTR]))(
+                frame, cookie, this, op_ret, op_errno,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FSETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fsetxattr_t)(priv->fops[GLUPY_FSETXATTR]))(
+                frame, this, fd, dict, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fsetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_fsetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fsetxattr_cbk, xl, xl->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+}
+
+
+void
+unwind_fsetxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_fsetxattr_fop (long py_this, fop_fsetxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FSETXATTR] = (long)fop;
+}
+
+void
+set_fsetxattr_cbk (long py_this, fop_fsetxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FSETXATTR] = (long)cbk;
+}
+
+/* FOP: FGETXATTR */
+
+int32_t
+glupy_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FGETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fgetxattr_cbk_t)(priv->cbks[GLUPY_FGETXATTR]))(
+                frame, cookie, this, op_ret, op_errno, dict,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FGETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fgetxattr_t)(priv->fops[GLUPY_FGETXATTR]))(
+                frame, this, fd, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fgetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_fgetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fgetxattr_cbk, xl, xl->fops->fgetxattr,
+                    fd, name, xdata);
+}
+
+
+void
+unwind_fgetxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict,
+                  dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+
+}
+
+
+void
+set_fgetxattr_fop (long py_this, fop_fgetxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FGETXATTR] = (long)fop;
+}
+
+
+void
+set_fgetxattr_cbk (long py_this, fop_fgetxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FGETXATTR] = (long)cbk;
+}
+
+/* FOP:REMOVEXATTR */
+
+int32_t
+glupy_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_REMOVEXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_removexattr_cbk_t)(priv->cbks[GLUPY_REMOVEXATTR]))(
+                frame, cookie, this, op_ret, op_errno, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_REMOVEXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_removexattr_t)(priv->fops[GLUPY_REMOVEXATTR]))(
+                frame, this, loc, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_removexattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+                  const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_removexattr_cbk, xl, xl->fops->removexattr,
+                    loc, name, xdata);
+}
+
+
+void
+unwind_removexattr (call_frame_t *frame, long cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_removexattr_fop (long py_this, fop_removexattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_REMOVEXATTR] = (long)fop;
+}
+
+void
+set_removexattr_cbk (long py_this, fop_removexattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_REMOVEXATTR] = (long)cbk;
+}
+
+
+/* FOP:FREMOVEXATTR */
+
+int32_t
+glupy_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FREMOVEXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fremovexattr_cbk_t)(priv->cbks[GLUPY_FREMOVEXATTR]))(
+                frame, cookie, this, op_ret, op_errno, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FREMOVEXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fremovexattr_t)(priv->fops[GLUPY_FREMOVEXATTR]))(
+                frame, this, fd, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fremovexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_fremovexattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fremovexattr_cbk, xl, xl->fops->fremovexattr,
+                    fd, name, xdata);
+}
+
+
+void
+unwind_fremovexattr (call_frame_t *frame, long cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_fremovexattr_fop (long py_this, fop_fremovexattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FREMOVEXATTR] = (long)fop;
+}
+
+void
+set_fremovexattr_cbk (long py_this, fop_fremovexattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FREMOVEXATTR] = (long)cbk;
+}
+
+
+/* FOP: LINK*/
+int32_t
+glupy_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_LINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_link_cbk_t)(priv->cbks[GLUPY_LINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_LINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_link_t)(priv->fops[GLUPY_LINK]))(
+                frame, this, oldloc, newloc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_link_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc,
+                    xdata);
+        return 0;
+}
+
+void
+wind_link (call_frame_t *frame, xlator_t *xl, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_link_cbk, xl, xl->fops->link,
+                    oldloc, newloc, xdata);
+}
+
+void
+unwind_link (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, inode_t *inode,
+             struct iatt *buf, struct iatt *preparent,
+             struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_link_fop (long py_this, fop_link_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_LINK] = (long)fop;
+}
+
+void
+set_link_cbk (long py_this, fop_link_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_LINK] = (long)cbk;
+}
+
+/* FOP: SYMLINK*/
+int32_t
+glupy_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_SYMLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_symlink_cbk_t)(priv->cbks[GLUPY_SYMLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_SYMLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_symlink_t)(priv->fops[GLUPY_SYMLINK]))(
+                frame, this, linkname, loc, umask, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_symlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->symlink, linkname, loc,
+                    umask, xdata);
+        return 0;
+}
+
+void
+wind_symlink (call_frame_t *frame, xlator_t *xl, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_symlink_cbk, xl, xl->fops->symlink,
+                    linkname, loc, umask, xdata);
+}
+
+void
+unwind_symlink (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_symlink_fop (long py_this, fop_symlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_SYMLINK] = (long)fop;
+}
+
+void
+set_symlink_cbk (long py_this, fop_symlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_SYMLINK] = (long)cbk;
+}
+
+
+/* FOP: READLINK */
+int32_t
+glupy_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, const char *path,
+                    struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readlink_cbk_t)(priv->cbks[GLUPY_READLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                path, buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path,
+                             buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                size_t size, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readlink_t)(priv->fops[GLUPY_READLINK]))(
+                frame, this, loc, size, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readlink, loc,
+                    size, xdata);
+        return 0;
+}
+
+void
+wind_readlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               size_t size, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_readlink_cbk, xl, xl->fops->readlink,
+                    loc, size, xdata);
+}
+
+void
+unwind_readlink (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *path,
+                 struct iatt *buf, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf,
+                             xdata);
+}
+
+void
+set_readlink_fop (long py_this, fop_readlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READLINK] = (long)fop;
+}
+
+void
+set_readlink_cbk (long py_this, fop_readlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READLINK] = (long)cbk;
+}
+
+
+/* FOP: UNLINK */
+
+int32_t
+glupy_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_UNLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_unlink_cbk_t)(priv->cbks[GLUPY_UNLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              int xflags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_UNLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_unlink_t)(priv->fops[GLUPY_UNLINK]))(
+                frame, this, loc, xflags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, loc,
+                    xflags, xdata);
+        return 0;
+}
+
+void
+wind_unlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+             int xflags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_unlink_cbk, xl, xl->fops->unlink,
+                    loc, xflags, xdata);
+}
+
+void
+unwind_unlink (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iatt *preparent, struct iatt *postparent,
+               dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+}
+
+void
+set_unlink_fop (long py_this, fop_unlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_UNLINK] = (long)fop;
+}
+
+void
+set_unlink_cbk (long py_this, fop_unlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_UNLINK] = (long)cbk;
+}
+
+
+/* FOP: MKDIR */
+
+int32_t
+glupy_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_MKDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_mkdir_cbk_t)(priv->cbks[GLUPY_MKDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_MKDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_mkdir_t)(priv->fops[GLUPY_MKDIR]))(
+                frame, this, loc, mode, umask, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                    xdata);
+        return 0;
+}
+
+void
+wind_mkdir (call_frame_t *frame, xlator_t *xl, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
+{
+
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_mkdir_cbk, xl, xl->fops->mkdir,
+                    loc, mode, umask, xdata);
+}
+
+void
+unwind_mkdir (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, inode_t *inode,
+              struct iatt *buf, struct iatt *preparent,
+              struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_mkdir_fop (long py_this, fop_mkdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_MKDIR] = (long)fop;
+}
+
+void
+set_mkdir_cbk (long py_this, fop_mkdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_MKDIR] = (long)cbk;
+}
+
+
+/* FOP: RMDIR */
+
+int32_t
+glupy_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_RMDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_rmdir_cbk_t)(priv->cbks[GLUPY_RMDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             int xflags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_RMDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_rmdir_t)(priv->fops[GLUPY_RMDIR]))(
+                frame, this, loc, xflags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_rmdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rmdir, loc,
+                    xflags, xdata);
+        return 0;
+}
+
+void
+wind_rmdir (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+            int xflags, dict_t *xdata)
+{
+
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_rmdir_cbk, xl, xl->fops->rmdir,
+                    loc, xflags, xdata);
+}
+
+void
+unwind_rmdir (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno,
+              struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+}
+
+void
+set_rmdir_fop (long py_this, fop_rmdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_RMDIR] = (long)fop;
+}
+
+void
+set_rmdir_cbk (long py_this, fop_rmdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_RMDIR] = (long)cbk;
+}
+
+
+/* NON-FOP-SPECIFIC CODE */
+
+
+long
+get_id (call_frame_t *frame)
+{
+        return (long)(frame->local);
+}
+
+uint64_t
+get_rootunique (call_frame_t *frame)
+{
+        return frame->root->unique;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        glupy_private_t         *priv           = NULL;
+        char                    *module_name    = NULL;
+        PyObject                *py_mod_name    = NULL;
+        PyObject                *py_init_func   = NULL;
+        PyObject                *py_args        = NULL;
+        PyObject                *syspath        = NULL;
+	PyObject                *path           = NULL;
+	static gf_boolean_t      py_inited      = _gf_false;
+        void *                   err_cleanup    = &&err_return;
+
+        if (dict_get_str(this->options,"module-name",&module_name) != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "missing module-name");
+                return -1;
+        }
+
+	priv = GF_CALLOC (1, sizeof (glupy_private_t), gf_glupy_mt_priv);
+        if (!priv) {
+                goto *err_cleanup;
+        }
+        this->private = priv;
+        err_cleanup = &&err_free_priv;
+
+	if (!py_inited) {
+                Py_Initialize();
+                PyEval_InitThreads();
+#if 0
+                (void)pthread_key_create(&gil_init_key,NULL);
+                (void)pthread_setspecific(gil_init_key,(void *)1);
+#endif
+                /* PyEval_InitThreads takes this "for" us.  No thanks. */
+                PyEval_ReleaseLock();
+                py_inited = _gf_true;
+        }
+
+	/* Adjust python's path */
+	syspath = PySys_GetObject("path");
+	path = PyString_FromString(GLUSTER_PYTHON_PATH);
+	PyList_Append(syspath, path);
+	Py_DECREF(path);
+
+	py_mod_name = PyString_FromString(module_name);
+        if (!py_mod_name) {
+                gf_log (this->name, GF_LOG_ERROR, "could not create name");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+
+        gf_log (this->name, GF_LOG_ERROR, "py_mod_name = %s", module_name);
+        priv->py_module = PyImport_Import(py_mod_name);
+        Py_DECREF(py_mod_name);
+        if (!priv->py_module) {
+                gf_log (this->name, GF_LOG_ERROR, "Python import failed");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        err_cleanup = &&err_deref_module;
+
+        py_init_func = PyObject_GetAttrString(priv->py_module, "xlator");
+        if (!py_init_func || !PyCallable_Check(py_init_func)) {
+                gf_log (this->name, GF_LOG_ERROR, "missing init func");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        err_cleanup = &&err_deref_init;
+
+        py_args = PyTuple_New(1);
+        if (!py_args) {
+                gf_log (this->name, GF_LOG_ERROR, "could not create args");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        PyTuple_SetItem(py_args,0,PyLong_FromLong((long)this));
+
+        /* TBD: pass in list of children */
+        priv->py_xlator = PyObject_CallObject(py_init_func, py_args);
+        Py_DECREF(py_args);
+        if (!priv->py_xlator) {
+                gf_log (this->name, GF_LOG_ERROR, "Python init failed");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        gf_log (this->name, GF_LOG_INFO, "init returned %p", priv->py_xlator);
+
+        return 0;
+
+err_deref_init:
+        Py_DECREF(py_init_func);
+err_deref_module:
+        Py_DECREF(priv->py_module);
+err_free_priv:
+        GF_FREE(priv);
+err_return:
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        glupy_private_t *priv = this->private;
+
+        if (!priv)
+                return;
+        Py_DECREF(priv->py_xlator);
+        Py_DECREF(priv->py_module);
+        this->private = NULL;
+        GF_FREE (priv);
+
+        return;
+}
+
+struct xlator_fops fops = {
+        .lookup       = glupy_lookup,
+        .create       = glupy_create,
+        .open         = glupy_open,
+        .readv        = glupy_readv,
+        .writev       = glupy_writev,
+        .opendir      = glupy_opendir,
+        .readdir      = glupy_readdir,
+        .stat         = glupy_stat,
+        .fstat        = glupy_fstat,
+        .setxattr     = glupy_setxattr,
+        .getxattr     = glupy_getxattr,
+        .fsetxattr    = glupy_fsetxattr,
+        .fgetxattr    = glupy_fgetxattr,
+        .removexattr  = glupy_removexattr,
+        .fremovexattr = glupy_fremovexattr,
+        .link         = glupy_link,
+        .unlink       = glupy_unlink,
+        .readlink     = glupy_readlink,
+        .symlink      = glupy_symlink,
+        .mkdir        = glupy_mkdir,
+        .rmdir        = glupy_rmdir,
+        .statfs       = glupy_statfs,
+        .readdirp     = glupy_readdirp
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/glupy/src/glupy.h b/xlators/features/glupy/src/glupy.h
new file mode 100644
index 000000000..8661fce88
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.h
@@ -0,0 +1,69 @@
+/*
+   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   This file is part of GlusterFS.
+
+   GlusterFS is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published
+   by the Free Software Foundation; either version 3 of the License,
+   or (at your option) any later version.
+
+   GlusterFS is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __GLUPY_H__
+#define __GLUPY_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include "mem-types.h"
+
+enum {
+        GLUPY_LOOKUP = 0,
+        GLUPY_CREATE,
+        GLUPY_OPEN,
+        GLUPY_READV,
+        GLUPY_WRITEV,
+        GLUPY_OPENDIR,
+        GLUPY_READDIR,
+        GLUPY_READDIRP,
+        GLUPY_STAT,
+        GLUPY_FSTAT,
+        GLUPY_STATFS,
+        GLUPY_SETXATTR,
+        GLUPY_GETXATTR,
+        GLUPY_FSETXATTR,
+        GLUPY_FGETXATTR,
+        GLUPY_REMOVEXATTR,
+        GLUPY_FREMOVEXATTR,
+        GLUPY_LINK,
+        GLUPY_UNLINK,
+        GLUPY_READLINK,
+        GLUPY_SYMLINK,
+        GLUPY_MKNOD,
+        GLUPY_MKDIR,
+        GLUPY_RMDIR,
+        GLUPY_N_FUNCS
+};
+
+typedef struct {
+        PyObject        *py_module;
+        PyObject        *py_xlator;
+        long            fops[GLUPY_N_FUNCS];
+        long            cbks[GLUPY_N_FUNCS];
+} glupy_private_t;
+
+enum gf_glupy_mem_types_ {
+        gf_glupy_mt_priv = gf_common_mt_end + 1,
+        gf_glupy_mt_end
+};
+
+#endif /* __GLUPY_H__ */
diff --git a/xlators/features/glupy/src/gluster.py b/xlators/features/glupy/src/gluster.py
new file mode 100644
index 000000000..a5daa77d3
--- /dev/null
+++ b/xlators/features/glupy/src/gluster.py
@@ -0,0 +1,841 @@
+import sys
+from ctypes import *
+
+dl = CDLL("",RTLD_GLOBAL)
+
+
+class call_frame_t (Structure):
+        pass
+
+class dev_t (Structure):
+        pass
+
+
+class dict_t (Structure):
+        pass
+
+
+class gf_dirent_t (Structure):
+        pass
+
+
+class iobref_t (Structure):
+	pass
+
+
+class iovec_t (Structure):
+        pass
+
+
+class list_head (Structure):
+        pass
+
+list_head._fields_ = [
+                ("next", POINTER(list_head)),
+                ("prev", POINTER(list_head))
+        ]
+
+
+class rwxperm_t (Structure):
+        _fields_ = [
+                ("read", c_uint8, 1),
+                ("write", c_uint8, 1),
+                ("execn", c_uint8, 1)
+        ]
+
+
+class statvfs_t (Structure):
+        pass
+
+
+class xlator_t (Structure):
+        pass
+
+
+class ia_prot_t (Structure):
+        _fields_ = [
+                ("suid", c_uint8, 1),
+                ("sgid", c_uint8, 1),
+                ("sticky", c_uint8, 1),
+                ("owner", rwxperm_t),
+                ("group", rwxperm_t),
+                ("other", rwxperm_t)
+        ]
+
+# For checking file type.
+(IA_INVAL, IA_IFREG, IA_IFDIR, IA_IFLNK, IA_IFBLK, IA_IFCHR, IA_IFIFO,
+ IA_IFSOCK) = xrange(8)
+
+
+class iatt_t (Structure):
+        _fields_ = [
+                ("ia_no", c_uint64),
+                ("ia_gfid", c_ubyte * 16),
+                ("ia_dev", c_uint64),
+                ("ia_type", c_uint),
+                ("ia_prot", ia_prot_t),
+                ("ia_nlink", c_uint32),
+                ("ia_uid", c_uint32),
+                ("ia_gid", c_uint32),
+                ("ia_rdev", c_uint64),
+                ("ia_size", c_uint64),
+                ("ia_blksize", c_uint32),
+                ("ia_blocks", c_uint64),
+                ("ia_atime", c_uint32 ),
+                ("ia_atime_nsec", c_uint32),
+                ("ia_mtime", c_uint32),
+                ("ia_mtime_nsec", c_uint32),
+                ("ia_ctime", c_uint32),
+                ("ia_ctime_nsec", c_uint32)
+        ]
+
+
+class mem_pool (Structure):
+        _fields_ = [
+                ("list", list_head),
+                ("hot_count", c_int),
+                ("cold_count", c_int),
+                ("lock", c_void_p),
+                ("padded_sizeof_type", c_ulong),
+                ("pool", c_void_p),
+                ("pool_end", c_void_p),
+                ("real_sizeof_type", c_int),
+                ("alloc_count", c_uint64),
+                ("pool_misses", c_uint64),
+                ("max_alloc", c_int),
+                ("curr_stdalloc", c_int),
+                ("max_stdalloc", c_int),
+                ("name", c_char_p),
+                ("global_list", list_head)
+        ]
+
+
+class U_ctx_key_inode (Union):
+        _fields_ = [
+                ("key", c_uint64),
+                ("xl_key", POINTER(xlator_t))
+        ]
+
+
+class U_ctx_value1 (Union):
+        _fields_ = [
+                ("value1", c_uint64),
+                ("ptr1", c_void_p)
+        ]
+
+
+class U_ctx_value2 (Union):
+        _fields_ = [
+                ("value2", c_uint64),
+                ("ptr2", c_void_p)
+        ]
+
+class inode_ctx (Structure):
+        _anonymous_ = ("u_key","u_value1","u_value2",)
+        _fields_ = [
+                ("u_key", U_ctx_key_inode),
+                ("u_value1", U_ctx_value1),
+                ("u_value2", U_ctx_value2)
+        ]
+
+class inode_t (Structure):
+        pass
+
+class inode_table_t (Structure):
+        _fields_ = [
+                ("lock", c_void_p),
+                ("hashsize", c_size_t),
+                ("name", c_char_p),
+                ("root", POINTER(inode_t)),
+                ("xl", POINTER(xlator_t)),
+                ("lru_limit", c_uint32),
+                ("inode_hash", POINTER(list_head)),
+                ("name_hash", POINTER(list_head)),
+                ("active", list_head),
+                ("active_size", c_uint32),
+                ("lru", list_head),
+                ("lru_size", c_uint32),
+                ("purge", list_head),
+                ("purge_size", c_uint32),
+                ("inode_pool", POINTER(mem_pool)),
+                ("dentry_pool", POINTER(mem_pool)),
+                ("fd_mem_pool", POINTER(mem_pool))
+        ]
+
+inode_t._fields_ = [
+                ("table", POINTER(inode_table_t)),
+                ("gfid", c_ubyte * 16),
+                ("lock", c_void_p),
+                ("nlookup", c_uint64),
+                ("fd_count", c_uint32),
+                ("ref", c_uint32),
+                ("ia_type", c_uint),
+                ("fd_list", list_head),
+                ("dentry_list", list_head),
+                ("hashv", list_head),
+                ("listv", list_head),
+                ("ctx", POINTER(inode_ctx))
+        ]
+
+
+
+class U_ctx_key_fd (Union):
+        _fields_ = [
+                ("key", c_uint64),
+                ("xl_key", c_void_p)
+        ]
+
+class fd_lk_ctx (Structure):
+        _fields_ = [
+                ("lk_list", list_head),
+                ("ref", c_int),
+                ("lock", c_void_p)
+        ]
+
+class fd_ctx (Structure):
+        _anonymous_ = ("u_key","u_value1")
+        _fields_ = [
+                ("u_key", U_ctx_key_fd),
+                ("u_value1", U_ctx_value1)
+        ]
+
+class fd_t (Structure):
+        _fields_ = [
+                ("pid", c_uint64),
+                ("flags", c_int32),
+                ("refcount", c_int32),
+                ("inode_list", list_head),
+                ("inode", POINTER(inode_t)),
+                ("lock", c_void_p),
+                ("ctx", POINTER(fd_ctx)),
+                ("xl_count", c_int),
+                ("lk_ctx", POINTER(fd_lk_ctx)),
+                ("anonymous", c_uint)
+        ]
+
+class loc_t (Structure):
+        _fields_ = [
+                ("path", c_char_p),
+                ("name", c_char_p),
+                ("inode", POINTER(inode_t)),
+                ("parent", POINTER(inode_t)),
+                ("gfid", c_ubyte * 16),
+                ("pargfid", c_ubyte * 16),
+        ]
+
+
+
+def _init_op (a_class, fop, cbk, wind, unwind):
+        # Decorators, used by translators. We could pass the signatures as
+        # parameters, but it's actually kind of nice to keep them around for
+        # inspection.
+        a_class.fop_type = apply(CFUNCTYPE,a_class.fop_sig)
+        a_class.cbk_type = apply(CFUNCTYPE,a_class.cbk_sig)
+        # Dispatch-function registration.
+        fop.restype = None
+        fop.argtypes = [ c_long, a_class.fop_type ]
+        # Callback-function registration.
+        cbk.restype = None
+        cbk.argtypes = [ c_long, a_class.cbk_type ]
+        # STACK_WIND function.
+        wind.restype = None
+        wind.argtypes = list(a_class.fop_sig[1:])
+        # STACK_UNWIND function.
+        unwind.restype = None
+        unwind.argtypes = list(a_class.cbk_sig[1:])
+
+class OpLookup:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(dict_t), POINTER(iatt_t))
+_init_op (OpLookup, dl.set_lookup_fop, dl.set_lookup_cbk,
+                    dl.wind_lookup,    dl.unwind_lookup)
+
+class OpCreate:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, c_uint, c_uint, POINTER(fd_t),
+                   POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(inode_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpCreate, dl.set_create_fop, dl.set_create_cbk,
+                    dl.wind_create,    dl.unwind_create)
+
+class OpOpen:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(fd_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
+_init_op (OpOpen, dl.set_open_fop, dl.set_open_cbk,
+                  dl.wind_open,    dl.unwind_open)
+
+class OpReadv:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, c_uint32, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iovec_t), c_int, POINTER(iatt_t),
+                   POINTER(iobref_t), POINTER(dict_t))
+_init_op (OpReadv, dl.set_readv_fop, dl.set_readv_cbk,
+                   dl.wind_readv,    dl.unwind_readv)
+class OpWritev:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(iovec_t), c_int, c_long, c_uint32,
+                   POINTER(iobref_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpWritev, dl.set_writev_fop, dl.set_writev_cbk,
+                    dl.wind_writev,    dl.unwind_writev)
+
+class OpOpendir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(fd_t) ,POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
+_init_op (OpOpendir, dl.set_opendir_fop, dl.set_opendir_cbk,
+                     dl.wind_opendir,    dl.unwind_opendir)
+
+class OpReaddir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
+_init_op (OpReaddir, dl.set_readdir_fop, dl.set_readdir_cbk,
+                     dl.wind_readdir,    dl.unwind_readdir)
+
+class OpReaddirp:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
+_init_op (OpReaddirp, dl.set_readdirp_fop, dl.set_readdirp_cbk,
+                      dl.wind_readdirp,    dl.unwind_readdirp)
+
+class OpStat:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpStat, dl.set_stat_fop, dl.set_stat_cbk,
+                  dl.wind_stat,    dl.unwind_stat)
+
+class OpFstat:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpFstat, dl.set_fstat_fop, dl.set_fstat_cbk,
+                   dl.wind_fstat,    dl.unwind_fstat)
+
+class OpStatfs:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(statvfs_t), POINTER(dict_t))
+_init_op (OpStatfs, dl.set_statfs_fop, dl.set_statfs_cbk,
+                    dl.wind_statfs,    dl.unwind_statfs)
+
+
+class OpSetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t), c_int32,
+                   POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpSetxattr, dl.set_setxattr_fop, dl.set_setxattr_cbk,
+                      dl.wind_setxattr,    dl.unwind_setxattr)
+
+class OpGetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_char_p, POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
+_init_op (OpGetxattr, dl.set_getxattr_fop, dl.set_getxattr_cbk,
+                      dl.wind_getxattr,    dl.unwind_getxattr)
+
+class OpFsetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(dict_t), c_int32,
+                   POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpFsetxattr, dl.set_fsetxattr_fop, dl.set_fsetxattr_cbk,
+                       dl.wind_fsetxattr,    dl.unwind_fsetxattr)
+
+class OpFgetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_char_p, POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
+_init_op (OpFgetxattr, dl.set_fgetxattr_fop, dl.set_fgetxattr_cbk,
+                       dl.wind_fgetxattr,    dl.unwind_fgetxattr)
+
+class OpRemovexattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_char_p, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpRemovexattr, dl.set_removexattr_fop, dl.set_removexattr_cbk,
+                         dl.wind_removexattr,    dl.unwind_removexattr)
+
+
+class OpFremovexattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_char_p, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpFremovexattr, dl.set_fremovexattr_fop, dl.set_fremovexattr_cbk,
+                          dl.wind_fremovexattr,    dl.unwind_fremovexattr)
+
+class OpLink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpLink, dl.set_link_fop, dl.set_link_cbk,
+                  dl.wind_link,    dl.unwind_link)
+
+class OpSymlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   c_char_p, POINTER(loc_t), c_uint, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpSymlink, dl.set_symlink_fop, dl.set_symlink_cbk,
+                     dl.wind_symlink,    dl.unwind_symlink)
+
+class OpUnlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpUnlink, dl.set_unlink_fop, dl.set_unlink_cbk,
+                    dl.wind_unlink,    dl.unwind_unlink)
+
+class OpReadlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_size_t, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, c_char_p, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpReadlink, dl.set_readlink_fop, dl.set_readlink_cbk,
+                      dl.wind_readlink,    dl.unwind_readlink)
+
+class OpMkdir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_uint, c_uint, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpMkdir, dl.set_mkdir_fop, dl.set_mkdir_cbk,
+                   dl.wind_mkdir,    dl.unwind_mkdir)
+
+class OpRmdir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpRmdir, dl.set_rmdir_fop, dl.set_rmdir_cbk,
+                   dl.wind_rmdir,    dl.unwind_rmdir)
+
+
+class Translator:
+        def __init__ (self, c_this):
+                # This is only here to keep references to the stubs we create,
+                # because ctypes doesn't and glupy.so can't because it doesn't
+                # get a pointer to the actual Python object. It's a dictionary
+                # instead of a list in case we ever allow changing fops/cbks
+                # after initialization and need to look them up.
+                self.stub_refs = {}
+                funcs = dir(self.__class__)
+                if "lookup_fop" in funcs:
+                        @OpLookup.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.lookup_fop (frame, this, loc, xdata)
+                        self.stub_refs["lookup_fop"] = stub
+                        dl.set_lookup_fop(c_this,stub)
+                if "lookup_cbk" in funcs:
+                        @OpLookup.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, inode,
+                                  buf, xdata, postparent, s=self):
+                                return s.lookup_cbk(frame, cookie, this, op_ret,
+                                                    op_errno, inode, buf, xdata,
+                                                    postparent)
+                        self.stub_refs["lookup_cbk"] = stub
+                        dl.set_lookup_cbk(c_this,stub)
+                if "create_fop" in funcs:
+                        @OpCreate.fop_type
+                        def stub (frame, this, loc, flags, mode, umask, fd,
+                                  xdata, s=self):
+                                return s.create_fop (frame, this, loc, flags,
+                                                     mode, umask, fd, xdata)
+                        self.stub_refs["create_fop"] = stub
+                        dl.set_create_fop(c_this,stub)
+                if "create_cbk" in funcs:
+                        @OpCreate.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.create_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, fd,
+                                                     inode, buf, preparent,
+                                                     postparent, xdata)
+                        self.stub_refs["create_cbk"] = stub
+                        dl.set_create_cbk(c_this,stub)
+                if "open_fop" in funcs:
+                        @OpOpen.fop_type
+                        def stub (frame, this, loc, flags, fd,
+                                  xdata, s=self):
+                                return s.open_fop (frame, this, loc, flags,
+                                                   fd, xdata)
+                        self.stub_refs["open_fop"] = stub
+                        dl.set_open_fop(c_this,stub)
+                if "open_cbk" in funcs:
+                        @OpOpen.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  xdata, s=self):
+                                return s.open_cbk (frame, cookie, this,
+                                                   op_ret, op_errno, fd,
+                                                   xdata)
+                        self.stub_refs["open_cbk"] = stub
+                        dl.set_open_cbk(c_this,stub)
+                if "readv_fop" in funcs:
+                        @OpReadv.fop_type
+                        def stub (frame, this, fd, size, offset, flags,
+                                  xdata, s=self):
+                                return s.readv_fop (frame, this, fd, size,
+                                                    offset, flags, xdata)
+                        self.stub_refs["readv_fop"] = stub
+                        dl.set_readv_fop(c_this,stub)
+                if "readv_cbk" in funcs:
+                        @OpReadv.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  vector, count, stbuf, iobref, xdata,
+                                  s=self):
+                                return s.readv_cbk (frame, cookie, this,
+                                                    op_ret, op_errno, vector,
+                                                    count, stbuf, iobref,
+                                                    xdata)
+                        self.stub_refs["readv_cbk"] = stub
+                        dl.set_readv_cbk(c_this,stub)
+                if "writev_fop" in funcs:
+                        @OpWritev.fop_type
+                        def stub (frame, this, fd, vector, count,
+                                  offset, flags, iobref, xdata, s=self):
+                                return s.writev_fop (frame, this, fd, vector,
+                                                     count, offset, flags,
+                                                     iobref, xdata)
+                        self.stub_refs["writev_fop"] = stub
+                        dl.set_writev_fop(c_this,stub)
+                if "writev_cbk" in funcs:
+                        @OpWritev.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  prebuf, postbuf, xdata, s=self):
+                                return s.writev_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, prebuf,
+                                                     postbuf, xdata)
+                        self.stub_refs["writev_cbk"] = stub
+                        dl.set_writev_cbk(c_this,stub)
+                if "opendir_fop" in funcs:
+                        @OpOpendir.fop_type
+                        def stub (frame, this, loc, fd, xdata, s=self):
+                                return s.opendir_fop (frame, this, loc, fd,
+                                                      xdata)
+                        self.stub_refs["opendir_fop"] = stub
+                        dl.set_opendir_fop(c_this,stub)
+                if "opendir_cbk" in funcs:
+                        @OpOpendir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  xdata, s=self):
+                                return s.opendir_cbk(frame, cookie, this,
+                                                     op_ret, op_errno, fd,
+                                                     xdata)
+                        self.stub_refs["opendir_cbk"] = stub
+                        dl.set_opendir_cbk(c_this,stub)
+                if "readdir_fop" in funcs:
+                        @OpReaddir.fop_type
+                        def stub (frame, this, fd, size, offset, xdata, s=self):
+                                return s.readdir_fop (frame, this, fd, size,
+                                                      offset, xdata)
+                        self.stub_refs["readdir_fop"] = stub
+                        dl.set_readdir_fop(c_this,stub)
+                if "readdir_cbk" in funcs:
+                        @OpReaddir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  entries, xdata, s=self):
+                                return s.readdir_cbk(frame, cookie, this,
+                                                     op_ret, op_errno, entries,
+                                                     xdata)
+                        self.stub_refs["readdir_cbk"] = stub
+                        dl.set_readdir_cbk(c_this,stub)
+                if "readdirp_fop" in funcs:
+                        @OpReaddirp.fop_type
+                        def stub (frame, this, fd, size, offset, xdata, s=self):
+                                return s.readdirp_fop (frame, this, fd, size,
+                                                       offset, xdata)
+                        self.stub_refs["readdirp_fop"] = stub
+                        dl.set_readdirp_fop(c_this,stub)
+                if "readdirp_cbk" in funcs:
+                        @OpReaddirp.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  entries, xdata, s=self):
+                                return s.readdirp_cbk (frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       entries, xdata)
+                        self.stub_refs["readdirp_cbk"] = stub
+                        dl.set_readdirp_cbk(c_this,stub)
+                if "stat_fop" in funcs:
+                        @OpStat.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.stat_fop (frame, this, loc, xdata)
+                        self.stub_refs["stat_fop"] = stub
+                        dl.set_stat_fop(c_this,stub)
+                if "stat_cbk" in funcs:
+                        @OpStat.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.stat_cbk(frame, cookie, this, op_ret,
+                                                  op_errno, buf, xdata)
+                        self.stub_refs["stat_cbk"] = stub
+                        dl.set_stat_cbk(c_this,stub)
+                if "fstat_fop" in funcs:
+                        @OpFstat.fop_type
+                        def stub (frame, this, fd, xdata, s=self):
+                                return s.fstat_fop (frame, this, fd, xdata)
+                        self.stub_refs["fstat_fop"] = stub
+                        dl.set_fstat_fop(c_this,stub)
+                if "fstat_cbk" in funcs:
+                        @OpFstat.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.fstat_cbk(frame, cookie, this, op_ret,
+                                                   op_errno, buf, xdata)
+                        self.stub_refs["fstat_cbk"] = stub
+                        dl.set_fstat_cbk(c_this,stub)
+                if "statfs_fop" in funcs:
+                        @OpStatfs.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.statfs_fop (frame, this, loc, xdata)
+                        self.stub_refs["statfs_fop"] = stub
+                        dl.set_statfs_fop(c_this,stub)
+                if "statfs_cbk" in funcs:
+                        @OpStatfs.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.statfs_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, buf,
+                                                     xdata)
+                        self.stub_refs["statfs_cbk"] = stub
+                        dl.set_statfs_cbk(c_this,stub)
+                if "setxattr_fop" in funcs:
+                        @OpSetxattr.fop_type
+                        def stub (frame, this, loc, dictionary, flags, xdata,
+                                  s=self):
+                                return s.setxattr_fop (frame, this, loc,
+                                                       dictionary, flags,
+                                                       xdata)
+                        self.stub_refs["setxattr_fop"] = stub
+                        dl.set_setxattr_fop(c_this,stub)
+                if "setxattr_cbk" in funcs:
+                        @OpSetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
+                                  s=self):
+                                return s.setxattr_cbk(frame, cookie, this,
+                                                      op_ret, op_errno, xdata)
+                        self.stub_refs["setxattr_cbk"] = stub
+                        dl.set_setxattr_cbk(c_this,stub)
+                if "getxattr_fop" in funcs:
+                        @OpGetxattr.fop_type
+                        def stub (frame, this, loc, name, xdata, s=self):
+                                return s.getxattr_fop (frame, this, loc, name,
+                                                       xdata)
+                        self.stub_refs["getxattr_fop"] = stub
+                        dl.set_getxattr_fop(c_this,stub)
+                if "getxattr_cbk" in funcs:
+                        @OpGetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  dictionary, xdata, s=self):
+                                return s.getxattr_cbk(frame, cookie, this,
+                                                      op_ret, op_errno,
+                                                      dictionary, xdata)
+                        self.stub_refs["getxattr_cbk"] = stub
+                        dl.set_getxattr_cbk(c_this,stub)
+                if "fsetxattr_fop" in funcs:
+                        @OpFsetxattr.fop_type
+                        def stub (frame, this, fd, dictionary, flags, xdata,
+                                  s=self):
+                                return s.fsetxattr_fop (frame, this, fd,
+                                                        dictionary, flags,
+                                                        xdata)
+                        self.stub_refs["fsetxattr_fop"] = stub
+                        dl.set_fsetxattr_fop(c_this,stub)
+                if "fsetxattr_cbk" in funcs:
+                        @OpFsetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
+                                  s=self):
+                                return s.fsetxattr_cbk(frame, cookie, this,
+                                                       op_ret, op_errno, xdata)
+                        self.stub_refs["fsetxattr_cbk"] = stub
+                        dl.set_fsetxattr_cbk(c_this,stub)
+                if "fgetxattr_fop" in funcs:
+                        @OpFgetxattr.fop_type
+                        def stub (frame, this, fd, name, xdata, s=self):
+                                return s.fgetxattr_fop (frame, this, fd, name,
+                                                        xdata)
+                        self.stub_refs["fgetxattr_fop"] = stub
+                        dl.set_fgetxattr_fop(c_this,stub)
+                if "fgetxattr_cbk" in funcs:
+                        @OpFgetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  dictionary, xdata, s=self):
+                                return s.fgetxattr_cbk(frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       dictionary, xdata)
+                        self.stub_refs["fgetxattr_cbk"] = stub
+                        dl.set_fgetxattr_cbk(c_this,stub)
+                if "removexattr_fop" in funcs:
+                        @OpRemovexattr.fop_type
+                        def stub (frame, this, loc, name, xdata, s=self):
+                                return s.removexattr_fop (frame, this, loc,
+                                                          name, xdata)
+                        self.stub_refs["removexattr_fop"] = stub
+                        dl.set_removexattr_fop(c_this,stub)
+                if "removexattr_cbk" in funcs:
+                        @OpRemovexattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  xdata, s=self):
+                                return s.removexattr_cbk(frame, cookie, this,
+                                                         op_ret, op_errno,
+                                                         xdata)
+                        self.stub_refs["removexattr_cbk"] = stub
+                        dl.set_removexattr_cbk(c_this,stub)
+                if "fremovexattr_fop" in funcs:
+                        @OpFremovexattr.fop_type
+                        def stub (frame, this, fd, name, xdata, s=self):
+                                return s.fremovexattr_fop (frame, this, fd,
+                                                           name, xdata)
+                        self.stub_refs["fremovexattr_fop"] = stub
+                        dl.set_fremovexattr_fop(c_this,stub)
+                if "fremovexattr_cbk" in funcs:
+                        @OpFremovexattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  xdata, s=self):
+                                return s.fremovexattr_cbk(frame, cookie, this,
+                                                          op_ret, op_errno,
+                                                          xdata)
+                        self.stub_refs["fremovexattr_cbk"] = stub
+                        dl.set_fremovexattr_cbk(c_this,stub)
+                if "link_fop" in funcs:
+                        @OpLink.fop_type
+                        def stub (frame, this, oldloc, newloc,
+                                  xdata, s=self):
+                                return s.link_fop (frame, this, oldloc,
+                                                   newloc, xdata)
+                        self.stub_refs["link_fop"] = stub
+                        dl.set_link_fop(c_this,stub)
+                if "link_cbk" in funcs:
+                        @OpLink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.link_cbk (frame, cookie, this,
+                                                   op_ret, op_errno, inode,
+                                                   buf, preparent,
+                                                   postparent, xdata)
+                        self.stub_refs["link_cbk"] = stub
+                        dl.set_link_cbk(c_this,stub)
+                if "symlink_fop" in funcs:
+                        @OpSymlink.fop_type
+                        def stub (frame, this, linkname, loc,
+                                  umask, xdata, s=self):
+                                return s.symlink_fop (frame, this, linkname,
+                                                      loc, umask, xdata)
+                        self.stub_refs["symlink_fop"] = stub
+                        dl.set_symlink_fop(c_this,stub)
+                if "symlink_cbk" in funcs:
+                        @OpSymlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.symlink_cbk (frame, cookie, this,
+                                                      op_ret, op_errno, inode,
+                                                      buf, preparent,
+                                                      postparent, xdata)
+                        self.stub_refs["symlink_cbk"] = stub
+                        dl.set_symlink_cbk(c_this,stub)
+                if "unlink_fop" in funcs:
+                        @OpUnlink.fop_type
+                        def stub (frame, this, loc, xflags,
+                                  xdata, s=self):
+                                return s.unlink_fop (frame, this, loc,
+                                                     xflags, xdata)
+                        self.stub_refs["unlink_fop"] = stub
+                        dl.set_unlink_fop(c_this,stub)
+                if "unlink_cbk" in funcs:
+                        @OpUnlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  preparent, postparent, xdata, s=self):
+                                return s.unlink_cbk (frame, cookie, this,
+                                                     op_ret, op_errno,
+                                                     preparent, postparent,
+                                                     xdata)
+                        self.stub_refs["unlink_cbk"] = stub
+                        dl.set_unlink_cbk(c_this,stub)
+                if "readlink_fop" in funcs:
+                        @OpReadlink.fop_type
+                        def stub (frame, this, loc, size,
+                                  xdata, s=self):
+                                return s.readlink_fop (frame, this, loc,
+                                                       size, xdata)
+                        self.stub_refs["readlink_fop"] = stub
+                        dl.set_readlink_fop(c_this,stub)
+                if "readlink_cbk" in funcs:
+                        @OpReadlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  path, buf, xdata, s=self):
+                                return s.readlink_cbk (frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       path, buf, xdata)
+                        self.stub_refs["readlink_cbk"] = stub
+                        dl.set_readlink_cbk(c_this,stub)
+                if "mkdir_fop" in funcs:
+                        @OpMkdir.fop_type
+                        def stub (frame, this, loc, mode, umask, xdata,
+                                  s=self):
+                                return s.mkdir_fop (frame, this, loc, mode,
+                                                    umask, xdata)
+                        self.stub_refs["mkdir_fop"] = stub
+                        dl.set_mkdir_fop(c_this,stub)
+                if "mkdir_cbk" in funcs:
+                        @OpMkdir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, inode,
+                                  buf, preparent, postparent, xdata, s=self):
+                                return s.mkdir_cbk (frame, cookie, this,
+                                                    op_ret, op_errno, inode,
+                                                    buf, preparent,
+                                                    postparent, xdata)
+                        self.stub_refs["mkdir_cbk"] = stub
+                        dl.set_mkdir_cbk(c_this,stub)
+                if "rmdir_fop" in funcs:
+                        @OpRmdir.fop_type
+                        def stub (frame, this, loc, xflags,
+                                  xdata, s=self):
+                                return s.rmdir_fop (frame, this, loc,
+                                                    xflags, xdata)
+                        self.stub_refs["rmdir_fop"] = stub
+                        dl.set_rmdir_fop(c_this,stub)
+                if "rmdir_cbk" in funcs:
+                        @OpRmdir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  preparent, postparent, xdata, s=self):
+                                return s.rmdir_cbk (frame, cookie, this,
+                                                    op_ret, op_errno,
+                                                    preparent, postparent,
+                                                    xdata)
+                        self.stub_refs["rmdir_cbk"] = stub
+                        dl.set_rmdir_cbk(c_this,stub)
diff --git a/xlators/features/glupy/src/helloworld.py b/xlators/features/glupy/src/helloworld.py
new file mode 100644
index 000000000..8fe403711
--- /dev/null
+++ b/xlators/features/glupy/src/helloworld.py
@@ -0,0 +1,19 @@
+import sys
+from gluster import *
+
+class xlator (Translator):
+
+    def __init__(self, c_this):
+        Translator.__init__(self, c_this)
+
+    def lookup_fop(self, frame, this, loc, xdata):
+        print "Python xlator: Hello!"
+        dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
+        return 0
+
+    def lookup_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                   xdata, postparent):
+        print "Python xlator: Hello again!"
+        dl.unwind_lookup(frame, cookie, this, op_ret, op_errno, inode, buf,
+                         xdata, postparent)
+        return 0
diff --git a/xlators/features/glupy/src/negative.py b/xlators/features/glupy/src/negative.py
new file mode 100644
index 000000000..1023602b9
--- /dev/null
+++ b/xlators/features/glupy/src/negative.py
@@ -0,0 +1,92 @@
+import sys
+from uuid import UUID
+from gluster import *
+
+# Negative-lookup-caching example.  If a file wasn't there the last time we
+# looked, it's probably still not there.  This translator keeps track of
+# those failed lookups for us, and returns ENOENT without needing to pass the
+# call any further for repeated requests.
+
+# If we were doing this for real, we'd need separate caches for each xlator
+# instance.  The easiest way to do this would be to have xlator.__init__
+# "register" each instance in a module-global dict, with the key as the C
+# translator address and the value as the xlator object itself.  For testing
+# and teaching, it's sufficient just to have one cache.  The keys are parent
+# GFIDs, and the entries are lists of names within that parent that we know
+# don't exist.
+cache = {}
+
+# TBD: we need a better way of handling per-request data (frame->local in C).
+dl.get_id.restype = c_long
+dl.get_id.argtypes = [ POINTER(call_frame_t) ]
+
+def uuid2str (gfid):
+    return str(UUID(''.join(map("{0:02x}".format, gfid))))
+
+class xlator (Translator):
+
+    def __init__ (self, c_this):
+        self.requests = {}
+        Translator.__init__(self,c_this)
+
+    def lookup_fop (self, frame, this, loc, xdata):
+        pargfid = uuid2str(loc.contents.pargfid)
+        print "lookup FOP: %s:%s" % (pargfid, loc.contents.name)
+        # Check the cache.
+        if cache.has_key(pargfid):
+            if loc.contents.name in cache[pargfid]:
+                print "short-circuiting for %s:%s" % (pargfid,
+                    loc.contents.name)
+                dl.unwind_lookup(frame,0,this,-1,2,None,None,None,None)
+                return 0
+        key = dl.get_id(frame)
+        self.requests[key] = (pargfid, loc.contents.name[:])
+        # TBD: get real child xl from init, pass it here
+        dl.wind_lookup(frame,POINTER(xlator_t)(),loc,xdata)
+        return 0
+
+    def lookup_cbk (self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                    xdata, postparent):
+        print "lookup CBK: %d (%d)" % (op_ret, op_errno)
+        key = dl.get_id(frame)
+        pargfid, name = self.requests[key]
+        # Update the cache.
+        if op_ret == 0:
+            print "found %s, removing from cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].discard(name)
+        elif op_errno == 2:    # ENOENT
+            print "failed to find %s, adding to cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].add(name)
+            else:
+                cache[pargfid] = set([name])
+        del self.requests[key]
+        dl.unwind_lookup(frame,cookie,this,op_ret,op_errno,
+                         inode,buf,xdata,postparent)
+        return 0
+
+    def create_fop (self, frame, this, loc, flags, mode, umask, fd, xdata):
+        pargfid = uuid2str(loc.contents.pargfid)
+        print "create FOP: %s:%s" % (pargfid, loc.contents.name)
+        key = dl.get_id(frame)
+        self.requests[key] = (pargfid, loc.contents.name[:])
+        # TBD: get real child xl from init, pass it here
+        dl.wind_create(frame,POINTER(xlator_t)(),loc,flags,mode,umask,fd,xdata)
+        return 0
+
+    def create_cbk (self, frame, cookie, this, op_ret, op_errno, fd, inode,
+                    buf, preparent, postparent, xdata):
+        print "create CBK: %d (%d)" % (op_ret, op_errno)
+        key = dl.get_id(frame)
+        pargfid, name = self.requests[key]
+        # Update the cache.
+        if op_ret == 0:
+            print "created %s, removing from cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].discard(name)
+        del self.requests[key]
+        dl.unwind_create(frame,cookie,this,op_ret,op_errno,fd,inode,buf,
+                         preparent,postparent,xdata)
+        return 0
+
diff --git a/xlators/features/index/Makefile.am b/xlators/features/index/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/index/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/index/src/Makefile.am b/xlators/features/index/src/Makefile.am
new file mode 100644
index 000000000..73bb8972e
--- /dev/null
+++ b/xlators/features/index/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = index.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+index_la_LDFLAGS = -module -avoid-version
+
+index_la_SOURCES = index.c
+index_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = index.h index-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/index/src/index-mem-types.h b/xlators/features/index/src/index-mem-types.h
new file mode 100644
index 000000000..553d492df
--- /dev/null
+++ b/xlators/features/index/src/index-mem-types.h
@@ -0,0 +1,22 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QUIESCE_MEM_TYPES_H__
+#define __QUIESCE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_index_mem_types_ {
+        gf_index_mt_priv_t = gf_common_mt_end + 1,
+        gf_index_inode_ctx_t = gf_common_mt_end + 2,
+        gf_index_fd_ctx_t = gf_common_mt_end + 3,
+        gf_index_mt_end
+};
+#endif
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
new file mode 100644
index 000000000..9253120f3
--- /dev/null
+++ b/xlators/features/index/src/index.c
@@ -0,0 +1,1489 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "index.h"
+#include "options.h"
+#include "glusterfs3-xdr.h"
+#include "syncop.h"
+
+#define XATTROP_SUBDIR "xattrop"
+#define BASE_INDICES_HOLDER_SUBDIR "base_indices_holder"
+
+call_stub_t *
+__index_dequeue (struct list_head *callstubs)
+{
+        call_stub_t *stub = NULL;
+
+        if (!list_empty (callstubs)) {
+                stub = list_entry (callstubs->next, call_stub_t, list);
+                list_del_init (&stub->list);
+        }
+
+        return stub;
+}
+
+inline static void
+__index_enqueue (struct list_head *callstubs, call_stub_t *stub)
+{
+        list_add_tail (&stub->list, callstubs);
+}
+
+static void
+worker_enqueue (xlator_t *this, call_stub_t *stub)
+{
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        pthread_mutex_lock (&priv->mutex);
+        {
+                __index_enqueue (&priv->callstubs, stub);
+                pthread_cond_signal (&priv->cond);
+        }
+        pthread_mutex_unlock (&priv->mutex);
+}
+
+void *
+index_worker (void *data)
+{
+        index_priv_t     *priv = NULL;
+        xlator_t         *this = NULL;
+        call_stub_t      *stub = NULL;
+        int               ret = 0;
+
+        THIS = data;
+        this = data;
+        priv = this->private;
+
+        for (;;) {
+                pthread_mutex_lock (&priv->mutex);
+                {
+                        while (list_empty (&priv->callstubs)) {
+                                ret = pthread_cond_wait (&priv->cond,
+                                                         &priv->mutex);
+                        }
+
+                        stub = __index_dequeue (&priv->callstubs);
+                }
+                pthread_mutex_unlock (&priv->mutex);
+
+                if (stub) /* guard against spurious wakeups */
+                        call_resume (stub);
+        }
+
+        return NULL;
+}
+int
+__index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+{
+        int               ret = 0;
+        index_inode_ctx_t *ictx = NULL;
+        uint64_t          tmpctx = 0;
+
+        ret = __inode_ctx_get (inode, this, &tmpctx);
+        if (!ret) {
+                ictx = (index_inode_ctx_t*) (long) tmpctx;
+                goto out;
+        }
+        ictx = GF_CALLOC (1, sizeof (*ictx), gf_index_inode_ctx_t);
+        if (!ictx) {
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&ictx->callstubs);
+        ret = __inode_ctx_put (inode, this, (uint64_t)ictx);
+        if (ret) {
+                GF_FREE (ictx);
+                ictx = NULL;
+                goto out;
+        }
+out:
+        if (ictx)
+                *ctx = ictx;
+        return ret;
+}
+
+int
+index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+{
+        int               ret = 0;
+
+        LOCK (&inode->lock);
+        {
+                ret = __index_inode_ctx_get (inode, this, ctx);
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
+static void
+make_index_dir_path (char *base, const char *subdir,
+                     char *index_dir, size_t len)
+{
+        snprintf (index_dir, len, "%s/%s", base, subdir);
+}
+
+int
+index_dir_create (xlator_t *this, const char *subdir)
+{
+        int          ret = 0;
+        struct       stat st = {0};
+        char         fullpath[PATH_MAX] = {0};
+        char         path[PATH_MAX] = {0};
+        char         *dir = NULL;
+        index_priv_t *priv = NULL;
+        size_t       len = 0;
+        size_t       pathlen = 0;
+
+        priv = this->private;
+        make_index_dir_path (priv->index_basepath, subdir, fullpath,
+                             sizeof (fullpath));
+        ret = stat (fullpath, &st);
+        if (!ret) {
+                if (!S_ISDIR (st.st_mode))
+                        ret = -2;
+                goto out;
+        }
+
+        pathlen = strlen (fullpath);
+        if ((pathlen > 1) && fullpath[pathlen - 1] == '/')
+                fullpath[pathlen - 1] = '\0';
+        dir = strchr (fullpath, '/');
+        while (dir) {
+                dir = strchr (dir + 1, '/');
+                if (dir)
+                        len = pathlen - strlen (dir);
+                else
+                        len = pathlen;
+                strncpy (path, fullpath, len);
+                path[len] = '\0';
+                ret = mkdir (path, 0600);
+                if (ret && (errno != EEXIST))
+                        goto out;
+        }
+        ret = 0;
+out:
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to "
+                        "create (%s)", priv->index_basepath, subdir,
+                        strerror (errno));
+        } else if (ret == -2) {
+                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to create, "
+                        "path exists, not a directory ", priv->index_basepath,
+                        subdir);
+        }
+        return ret;
+}
+
+void
+index_get_index (index_priv_t *priv, uuid_t index)
+{
+        LOCK (&priv->lock);
+        {
+                uuid_copy (index, priv->index);
+        }
+        UNLOCK (&priv->lock);
+}
+
+void
+index_generate_index (index_priv_t *priv, uuid_t index)
+{
+        LOCK (&priv->lock);
+        {
+                //To prevent duplicate generates.
+                //This method fails if number of contending threads is greater
+                //than MAX_LINK count of the fs
+                if (!uuid_compare (priv->index, index))
+                        uuid_generate (priv->index);
+                uuid_copy (index, priv->index);
+        }
+        UNLOCK (&priv->lock);
+}
+
+static void
+make_index_path (char *base, const char *subdir, uuid_t index,
+                 char *index_path, size_t len)
+{
+        make_index_dir_path (base, subdir, index_path, len);
+        snprintf (index_path + strlen (index_path), len - strlen (index_path),
+                  "/%s-%s", subdir, uuid_utoa (index));
+}
+
+static void
+make_gfid_path (char *base, const char *subdir, uuid_t gfid,
+                char *gfid_path, size_t len)
+{
+        make_index_dir_path (base, subdir, gfid_path, len);
+        snprintf (gfid_path + strlen (gfid_path), len - strlen (gfid_path),
+                  "/%s", uuid_utoa (gfid));
+}
+
+static void
+make_file_path (char *base, const char *subdir, const char *filename,
+                char *file_path, size_t len)
+{
+        make_index_dir_path (base, subdir, file_path, len);
+        snprintf (file_path + strlen (file_path), len - strlen (file_path),
+                  "/%s", filename);
+}
+
+static void
+check_delete_stale_index_file (xlator_t *this, char *filename)
+{
+        int             ret = 0;
+        struct stat     st = {0};
+        struct stat     base_index_st = {0};
+        char            filepath[PATH_MAX] = {0};
+        char            filepath_under_base_indices_holder[PATH_MAX] = {0};
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        if (priv->to_be_healed_states != synced_state)
+                return;
+
+        make_file_path (priv->index_basepath, XATTROP_SUBDIR,
+                        filename, filepath, sizeof (filepath));
+
+        make_file_path (priv->index_basepath, BASE_INDICES_HOLDER_SUBDIR,
+                        filename, filepath_under_base_indices_holder,
+                        sizeof (filepath_under_base_indices_holder));
+
+
+        ret = stat (filepath_under_base_indices_holder, &base_index_st);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Base index is not created"
+                        "under index/base_indices_holder");
+                return;
+        }
+
+        ret = stat (filepath, &st);
+        if (!ret && st.st_nlink == 2) {
+                unlink (filepath);
+                unlink (filepath_under_base_indices_holder);
+        }
+}
+
+static int
+index_fill_readdir (fd_t *fd, DIR *dir, off_t off,
+                    size_t size, gf_dirent_t *entries, readdir_directory type)
+{
+        off_t     in_case = -1;
+        size_t    filled = 0;
+        int       count = 0;
+        char      entrybuf[sizeof(struct dirent) + 256 + 8];
+        struct dirent  *entry          = NULL;
+        int32_t              this_size      = -1;
+        gf_dirent_t          *this_entry     = NULL;
+        xlator_t             *this = NULL;
+
+        this = THIS;
+        if (!off) {
+                rewinddir (dir);
+        } else {
+                seekdir (dir, off);
+        }
+
+        while (filled <= size) {
+                in_case = telldir (dir);
+
+                if (in_case == -1) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "telldir failed on dir=%p: %s",
+                                dir, strerror (errno));
+                        goto out;
+                }
+
+                errno = 0;
+                entry = NULL;
+                readdir_r (dir, (struct dirent *)entrybuf, &entry);
+
+                if (!entry) {
+                        if (errno == EBADF) {
+                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        "readdir failed on dir=%p: %s",
+                                        dir, strerror (errno));
+                                goto out;
+                        }
+                        break;
+                }
+
+                if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
+                              strlen (XATTROP_SUBDIR"-")) &&
+                              (type == INDEX_XATTROP)) {
+                        check_delete_stale_index_file (this, entry->d_name);
+                        continue;
+                }
+
+                this_size = max (sizeof (gf_dirent_t),
+                                 sizeof (gfs3_dirplist))
+                        + strlen (entry->d_name) + 1;
+
+                if (this_size + filled > size) {
+                        seekdir (dir, in_case);
+                        break;
+                }
+
+                this_entry = gf_dirent_for_name (entry->d_name);
+
+                if (!this_entry) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "could not create gf_dirent for entry %s: (%s)",
+                                entry->d_name, strerror (errno));
+                        goto out;
+                }
+                this_entry->d_off = telldir (dir);
+                this_entry->d_ino = entry->d_ino;
+
+                list_add_tail (&this_entry->list, &entries->list);
+
+                filled += this_size;
+                count ++;
+        }
+
+        if ((!readdir (dir) && (errno == 0)))
+                /* Indicate EOF */
+                errno = ENOENT;
+out:
+        return count;
+}
+
+int
+sync_base_indices (void *index_priv)
+{
+        index_priv_t    *priv = NULL;
+        DIR             *dir_base_holder  = NULL;
+        DIR             *xattrop_dir = NULL;
+        struct dirent   *entry = NULL;
+        char            base_indices_holder[PATH_MAX] = {0};
+        char            xattrop_directory[PATH_MAX] = {0};
+        char            base_index_path[PATH_MAX] = {0};
+        char            xattrop_index_path[PATH_MAX] = {0};
+        int             ret = 0;
+
+        priv = index_priv;
+
+        snprintf (base_indices_holder, PATH_MAX, "%s/%s", priv->index_basepath,
+                  BASE_INDICES_HOLDER_SUBDIR);
+        snprintf (xattrop_directory, PATH_MAX, "%s/%s", priv->index_basepath,
+                  XATTROP_SUBDIR);
+
+        if ((dir_base_holder = opendir(base_indices_holder)) == NULL) {
+                ret = -1;
+                goto out;
+        }
+        if ((xattrop_dir = opendir (xattrop_directory)) == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        priv->to_be_healed_states = sync_started;
+        while ((entry = readdir(xattrop_dir)) != NULL) {
+                if (!strcmp (entry->d_name, ".") ||
+                    !strcmp (entry->d_name, "..")) {
+                        continue;
+                }
+                if (strncmp (entry->d_name, XATTROP_SUBDIR"-",
+                    strlen (XATTROP_SUBDIR"-"))) {
+                        continue;
+                }
+                if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
+                    strlen (XATTROP_SUBDIR"-"))) {
+
+                    snprintf (xattrop_index_path, PATH_MAX, "%s/%s",
+                              xattrop_directory, entry->d_name);
+
+                    snprintf (base_index_path, PATH_MAX, "%s/%s",
+                              base_indices_holder, entry->d_name);
+
+                    ret = link (xattrop_index_path, base_index_path);
+                    if (ret && errno != EEXIST)
+                        goto out;
+
+                }
+        }
+        ret = closedir (xattrop_dir);
+        if (ret)
+                goto out;
+        ret = closedir (dir_base_holder);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+int
+base_indices_syncing_done (int ret, call_frame_t *frame, void *data)
+{
+        index_priv_t         *priv = NULL;
+        priv = data;
+
+        if (!priv)
+                goto out;
+
+        if (ret) {
+                priv->to_be_healed_states = sync_not_started;
+        } else {
+                priv->to_be_healed_states = synced_state;
+        }
+
+        STACK_DESTROY (frame->root);
+
+out:
+        return 0;
+}
+
+int
+sync_base_indices_from_xattrop (xlator_t *this)
+{
+
+        index_priv_t          *priv = NULL;
+        char                  base_indices_holder[PATH_MAX] =  {0};
+        int                   ret = 0;
+        struct stat           st = {0};
+        DIR                   *dir = NULL;
+        struct dirent         *entry = NULL;
+        call_frame_t          *frame = NULL;
+
+        priv = this->private;
+
+        if (priv->to_be_healed_states != sync_not_started) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (base_indices_holder, PATH_MAX, "%s/%s", priv->index_basepath,
+                  BASE_INDICES_HOLDER_SUBDIR);
+
+        ret = stat (base_indices_holder, &st);
+
+        if (ret && (errno != ENOENT)) {
+                goto out;
+        } else if (errno == ENOENT) {
+                ret = index_dir_create (this, BASE_INDICES_HOLDER_SUBDIR);
+                if (ret)
+                        goto out;
+        } else {
+                if ((dir = opendir (base_indices_holder)) == NULL) {
+                        ret = -1;
+                        goto out;
+                }
+                while ((entry = readdir (dir)) != NULL) {
+                        if (!strcmp (entry->d_name, ".") ||
+                            !strcmp (entry->d_name,"..")) {
+                                continue;
+                        }
+                        ret = unlink (entry->d_name);
+                        if (ret)
+                                goto out;
+                }
+                closedir (dir);
+        }
+
+        /*At this point of time we have index/base_indicies_holder directory
+         *is with no entries*/
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto out;
+        }
+        set_lk_owner_from_ptr (&frame->root->lk_owner, frame->root);
+
+        frame->root->pid = LOW_PRIO_PROC_PID;
+
+        ret = synctask_new (this->ctx->env, sync_base_indices,
+                            base_indices_syncing_done,frame, priv);
+
+
+
+out:
+        return ret;
+
+}
+
+int
+index_add (xlator_t *this, uuid_t gfid, const char *subdir)
+{
+        int32_t           op_errno = 0;
+        char              gfid_path[PATH_MAX] = {0};
+        char              index_path[PATH_MAX] = {0};
+        char              base_path[PATH_MAX] = {0};
+        int               ret = 0;
+        uuid_t            index = {0};
+        index_priv_t      *priv = NULL;
+        struct stat       st = {0};
+        int               fd = 0;
+        int               index_created = 0;
+
+        priv = this->private;
+        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !uuid_is_null (gfid),
+                                       out, op_errno, EINVAL);
+
+        make_gfid_path (priv->index_basepath, subdir, gfid,
+                        gfid_path, sizeof (gfid_path));
+
+        ret = stat (gfid_path, &st);
+        if (!ret)
+                goto out;
+        index_get_index (priv, index);
+        make_index_path (priv->index_basepath, subdir,
+                         index, index_path, sizeof (index_path));
+        ret = link (index_path, gfid_path);
+        if (!ret || (errno == EEXIST))  {
+                ret = 0;
+                index_created = 1;
+                goto out;
+        }
+
+
+        op_errno = errno;
+        if (op_errno == ENOENT) {
+                ret = index_dir_create (this, subdir);
+                if (ret)
+                        goto out;
+        } else if (op_errno == EMLINK) {
+                index_generate_index (priv, index);
+                make_index_path (priv->index_basepath, subdir,
+                                 index, index_path, sizeof (index_path));
+        } else {
+                goto out;
+        }
+
+        fd = creat (index_path, 0);
+        if ((fd < 0) && (errno != EEXIST)) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
+                        "create index (%s)", uuid_utoa (gfid),
+                        strerror (errno));
+                goto out;
+        }
+
+        if (fd >= 0)
+                close (fd);
+
+        ret = link (index_path, gfid_path);
+        if (ret && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
+                        "add to index (%s)", uuid_utoa (gfid),
+                        strerror (errno));
+                goto out;
+        } else {
+                index_created = 1;
+        }
+
+        if (priv->to_be_healed_states != sync_not_started) {
+                 make_index_path (priv->index_basepath,
+                                  GF_BASE_INDICES_HOLDER_GFID,
+                                  index, base_path, sizeof (base_path));
+                 ret = link (index_path, base_path);
+                 if (ret)
+                         goto out;
+        }
+
+        ret = 0;
+out:
+        /*If base_indices_holder is not created: create and sync
+         *If directory is present: delete contents and start syncing
+         *If syncing is in progress :No need to do any thing
+         *If syncing is done: No need to do anything*/
+        if (!ret) {
+                switch (priv->to_be_healed_states) {
+                        case sync_not_started:
+                                ret = sync_base_indices_from_xattrop (this);
+                                break;
+                        case sync_started:
+                        case synced_state:
+                                /*No need to do anything*/
+                                break;
+                }
+        }
+        return ret;
+}
+
+int
+index_del (xlator_t *this, uuid_t gfid, const char *subdir)
+{
+        int32_t      op_errno __attribute__((unused)) = 0;
+        index_priv_t *priv = NULL;
+        int          ret = 0;
+        char         gfid_path[PATH_MAX] = {0};
+
+        priv = this->private;
+        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !uuid_is_null (gfid),
+                                       out, op_errno, EINVAL);
+        make_gfid_path (priv->index_basepath, subdir, gfid,
+                        gfid_path, sizeof (gfid_path));
+        ret = unlink (gfid_path);
+        if (ret && (errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to delete from index (%s)",
+                        gfid_path, strerror (errno));
+                ret = -errno;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+_check_key_is_zero_filled (dict_t *d, char *k, data_t *v,
+                           void *tmp)
+{
+        if (mem_0filled ((const char*)v->data, v->len)) {
+                /* -1 means, no more iterations, treat as 'break' */
+                return -1;
+        }
+        return 0;
+}
+
+
+void
+_xattrop_index_action (xlator_t *this, inode_t *inode,  dict_t *xattr)
+{
+        gf_boolean_t      zero_xattr = _gf_true;
+        index_inode_ctx_t *ctx = NULL;
+        int               ret = 0;
+
+        ret = dict_foreach (xattr, _check_key_is_zero_filled, NULL);
+        if (ret == -1)
+                zero_xattr = _gf_false;
+
+        ret = index_inode_ctx_get (inode, this, &ctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Not able to %s %s -> index",
+                        zero_xattr?"add":"del", uuid_utoa (inode->gfid));
+                goto out;
+        }
+        if (zero_xattr) {
+                if (ctx->state == NOTIN)
+                        goto out;
+                ret = index_del (this, inode->gfid, XATTROP_SUBDIR);
+                if (!ret)
+                        ctx->state = NOTIN;
+        } else {
+                if (ctx->state == IN)
+                        goto out;
+                ret = index_add (this, inode->gfid, XATTROP_SUBDIR);
+                if (!ret)
+                        ctx->state = IN;
+        }
+out:
+        return;
+}
+
+void
+fop_xattrop_index_action (xlator_t *this, inode_t *inode, dict_t *xattr)
+{
+        _xattrop_index_action (this, inode, xattr);
+}
+
+void
+fop_fxattrop_index_action (xlator_t *this, inode_t *inode, dict_t *xattr)
+{
+        _xattrop_index_action (this, inode, xattr);
+}
+
+inline gf_boolean_t
+index_xattrop_track (loc_t *loc, gf_xattrop_flags_t flags, dict_t *dict)
+{
+        return (flags == GF_XATTROP_ADD_ARRAY);
+}
+
+inline gf_boolean_t
+index_fxattrop_track (fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+{
+        return (flags == GF_XATTROP_ADD_ARRAY);
+}
+
+int
+__index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+{
+        int               ret = 0;
+        index_fd_ctx_t    *fctx = NULL;
+        uint64_t          tmpctx = 0;
+        char              index_dir[PATH_MAX] = {0};
+        index_priv_t      *priv = NULL;
+
+        priv = this->private;
+        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid)) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        ret = __fd_ctx_get (fd, this, &tmpctx);
+        if (!ret) {
+                fctx = (index_fd_ctx_t*) (long) tmpctx;
+                goto out;
+        }
+
+        fctx = GF_CALLOC (1, sizeof (*fctx), gf_index_fd_ctx_t);
+        if (!fctx) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
+                             index_dir, sizeof (index_dir));
+        fctx->dir = opendir (index_dir);
+        if (!fctx->dir) {
+                ret = -errno;
+                GF_FREE (fctx);
+                fctx = NULL;
+                goto out;
+        }
+
+        ret = __fd_ctx_set (fd, this, (uint64_t)(long)fctx);
+        if (ret) {
+                GF_FREE (fctx);
+                fctx = NULL;
+                ret = -EINVAL;
+                goto out;
+        }
+out:
+        if (fctx)
+                *ctx = fctx;
+        return ret;
+}
+
+int
+index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+{
+        int     ret = 0;
+        LOCK (&fd->lock);
+        {
+                ret = __index_fd_ctx_get (fd, this, ctx);
+        }
+        UNLOCK (&fd->lock);
+        return ret;
+}
+
+//new - Not NULL means start a fop
+//new - NULL means done processing the fop
+void
+index_queue_process (xlator_t *this, inode_t *inode, call_stub_t *new)
+{
+        call_stub_t       *stub = NULL;
+        index_inode_ctx_t *ctx = NULL;
+        int               ret = 0;
+        call_frame_t      *frame = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ret = __index_inode_ctx_get (inode, this, &ctx);
+                if (ret)
+                        goto unlock;
+
+                if (new) {
+                        __index_enqueue (&ctx->callstubs, new);
+                        new = NULL;
+                } else {
+                        ctx->processing = _gf_false;
+                }
+
+                if (!ctx->processing) {
+                        stub = __index_dequeue (&ctx->callstubs);
+                        if (stub)
+                                ctx->processing = _gf_true;
+                        else
+                                ctx->processing = _gf_false;
+                }
+        }
+unlock:
+        UNLOCK (&inode->lock);
+
+        if (ret && new) {
+                frame = new->frame;
+                if (new->fop == GF_FOP_XATTROP) {
+                        INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM,
+                                            NULL, NULL);
+                } else if (new->fop == GF_FOP_FXATTROP) {
+                        INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM,
+                                            NULL, NULL);
+                }
+                call_stub_destroy (new);
+        } else if (stub) {
+                call_resume (stub);
+        }
+        return;
+}
+
+int32_t
+index_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
+{
+        inode_t *inode = NULL;
+
+        inode = inode_ref (frame->local);
+        if (op_ret < 0)
+                goto out;
+        fop_xattrop_index_action (this, frame->local, xattr);
+out:
+        INDEX_STACK_UNWIND (xattrop, frame, op_ret, op_errno, xattr, xdata);
+        index_queue_process (this, inode, NULL);
+        inode_unref (inode);
+
+        return 0;
+}
+
+int32_t
+index_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                    dict_t *xdata)
+{
+        inode_t *inode = NULL;
+
+        inode = inode_ref (frame->local);
+        if (op_ret < 0)
+                goto out;
+
+        fop_fxattrop_index_action (this, frame->local, xattr);
+out:
+        INDEX_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, xattr, xdata);
+        index_queue_process (this, inode, NULL);
+        inode_unref (inode);
+
+        return 0;
+}
+
+int
+index_xattrop_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        STACK_WIND (frame, index_xattrop_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->xattrop, loc, optype, xattr,
+                    xdata);
+        return 0;
+}
+
+int
+index_fxattrop_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        STACK_WIND (frame, index_fxattrop_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fxattrop, fd, optype, xattr,
+                    xdata);
+        return 0;
+}
+
+int32_t
+index_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+	       gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+
+        if (!index_xattrop_track (loc, flags, dict))
+                goto out;
+
+        frame->local = inode_ref (loc->inode);
+        stub = fop_xattrop_stub (frame, index_xattrop_wrapper,
+                                 loc, flags, dict, xdata);
+        if (!stub) {
+                INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        index_queue_process (this, loc->inode, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+        return 0;
+}
+
+int32_t
+index_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+
+        if (!index_fxattrop_track (fd, flags, dict))
+                goto out;
+
+        frame->local = inode_ref (fd->inode);
+        stub = fop_fxattrop_stub (frame, index_fxattrop_wrapper,
+                                  fd, flags, dict, xdata);
+        if (!stub) {
+                INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM, NULL, xdata);
+                return 0;
+        }
+
+        index_queue_process (this, fd->inode, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+        return 0;
+}
+
+int32_t
+index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
+                        loc_t *loc, const char *name, dict_t *xdata)
+{
+        index_priv_t    *priv = NULL;
+        dict_t          *xattr = NULL;
+        int             ret = 0;
+
+        priv = this->private;
+
+        xattr = dict_new ();
+        if (!xattr) {
+                ret = -ENOMEM;
+                goto done;
+        }
+
+        if (!strcmp (name, GF_XATTROP_INDEX_GFID)) {
+
+                ret = dict_set_static_bin (xattr, (char*)name,
+                                           priv->xattrop_vgfid,
+                                           sizeof (priv->xattrop_vgfid));
+
+        } else if (!strcmp (name, GF_BASE_INDICES_HOLDER_GFID)) {
+
+                ret = dict_set_static_bin (xattr, (char*)name,
+                                           priv->base_indices_holder_vgfid,
+                                      sizeof (priv->base_indices_holder_vgfid));
+        }
+        if (ret) {
+                ret = -ENOMEM;
+                gf_log (THIS->name, GF_LOG_ERROR, "xattrop index "
+                        "gfid set failed");
+                goto done;
+        }
+done:
+        if (ret)
+                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, xattr, xdata);
+        else
+                STACK_UNWIND_STRICT (getxattr, frame, 0, 0, xattr, xdata);
+
+        if (xattr)
+                dict_unref (xattr);
+
+        return 0;
+}
+
+int32_t
+index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
+                      loc_t *loc, dict_t *xattr_req)
+{
+        index_priv_t    *priv = NULL;
+        struct stat     lstatbuf = {0};
+        int             ret = 0;
+        int32_t         op_errno = EINVAL;
+        int32_t         op_ret = -1;
+        char            path[PATH_MAX] = {0};
+        struct iatt     stbuf        = {0, };
+        struct iatt     postparent = {0,};
+        dict_t          *xattr = NULL;
+        gf_boolean_t    is_dir = _gf_false;
+
+        priv = this->private;
+
+        VALIDATE_OR_GOTO (loc, done);
+        if (!uuid_compare (loc->gfid, priv->xattrop_vgfid)) {
+                make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
+                                     path, sizeof (path));
+                is_dir = _gf_true;
+        } else if (!uuid_compare (loc->pargfid, priv->xattrop_vgfid)) {
+                make_file_path (priv->index_basepath, XATTROP_SUBDIR,
+                                loc->name, path, sizeof (path));
+        } else if (!uuid_compare (loc->gfid,priv->base_indices_holder_vgfid)){
+                make_index_dir_path (priv->index_basepath,
+                                     BASE_INDICES_HOLDER_SUBDIR, path,
+                                    sizeof (path));
+                is_dir = _gf_true;
+        } else if (!uuid_compare (loc->pargfid, priv->base_indices_holder_vgfid)) {
+                make_file_path (priv->index_basepath,
+                                BASE_INDICES_HOLDER_SUBDIR,loc->name, path,
+                                sizeof (path));
+        }
+
+        ret = lstat (path, &lstatbuf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on index dir "
+                        "(%s)", strerror (errno));
+                op_errno = errno;
+                goto done;
+        } else if (!S_ISDIR (lstatbuf.st_mode) && is_dir) {
+                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on index dir, "
+                        "not a directory");
+                op_errno = ENOENT;
+                goto done;
+        }
+        xattr = dict_new ();
+        if (!xattr) {
+                op_errno = ENOMEM;
+                goto done;
+        }
+
+        iatt_from_stat (&stbuf, &lstatbuf);
+        if (is_dir && !uuid_compare (loc->gfid, priv->xattrop_vgfid)) {
+                uuid_copy (stbuf.ia_gfid, priv->xattrop_vgfid);
+        } else if (is_dir &&
+                !uuid_compare (loc->gfid, priv->base_indices_holder_vgfid)) {
+                uuid_copy (stbuf.ia_gfid, priv->base_indices_holder_vgfid);
+        } else {
+                uuid_generate (stbuf.ia_gfid);
+        }
+        stbuf.ia_ino = -1;
+        op_ret = 0;
+done:
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno,
+                             loc->inode, &stbuf, xattr, &postparent);
+        if (xattr)
+                dict_unref (xattr);
+        return 0;
+}
+
+int32_t
+base_indices_readdir_wrapper (call_frame_t *frame, xlator_t *this,
+                              fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        index_priv_t    *priv = NULL;
+        char            base_indices_holder[PATH_MAX] = {0};
+        DIR             *dir = NULL;
+        int32_t         op_ret = -1;
+        int32_t         op_errno = 0;
+        int             count = 0;
+        gf_dirent_t     entries;
+
+        priv = this->private;
+
+        make_index_dir_path (priv->index_basepath, BASE_INDICES_HOLDER_SUBDIR,
+                             base_indices_holder, sizeof (base_indices_holder));
+
+        dir = opendir (base_indices_holder);
+        if (!dir) {
+                op_errno = EINVAL;
+                goto done;
+        }
+
+
+        INIT_LIST_HEAD (&entries.list);
+
+        count = index_fill_readdir (fd, dir, off, size, &entries,
+                                    BASE_INDICES_HOLDER);
+        /* pick ENOENT to indicate EOF */
+        op_errno = errno;
+        op_ret = count;
+        closedir (dir);
+done:
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
+        gf_dirent_free (&entries);
+        return 0;
+}
+
+int32_t
+index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
+                       fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        index_fd_ctx_t       *fctx           = NULL;
+        DIR                  *dir            = NULL;
+        int                   ret            = -1;
+        int32_t               op_ret         = -1;
+        int32_t               op_errno       = 0;
+        int                   count          = 0;
+        gf_dirent_t           entries;
+
+        INIT_LIST_HEAD (&entries.list);
+
+        ret = index_fd_ctx_get (fd, this, &fctx);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "pfd is NULL, fd=%p", fd);
+                op_errno = -ret;
+                goto done;
+        }
+
+        dir = fctx->dir;
+
+        if (!dir) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dir is NULL for fd=%p", fd);
+                op_errno = EINVAL;
+                goto done;
+        }
+
+        count = index_fill_readdir (fd, dir, off, size, &entries,
+                                    INDEX_XATTROP);
+
+        /* pick ENOENT to indicate EOF */
+        op_errno = errno;
+        op_ret = count;
+done:
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
+        gf_dirent_free (&entries);
+        return 0;
+}
+
+int
+index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+                      dict_t *xdata)
+{
+        index_priv_t    *priv = NULL;
+        int32_t         op_ret = 0;
+        int32_t         op_errno = 0;
+        int             ret = 0;
+        struct  iatt    preparent = {0};
+        struct  iatt    postparent = {0};
+        char            index_dir[PATH_MAX] = {0};
+        struct  stat    lstatbuf = {0};
+        uuid_t          gfid = {0};
+
+        priv = this->private;
+        make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
+                             index_dir, sizeof (index_dir));
+        ret = lstat (index_dir, &lstatbuf);
+        if (ret < 0) {
+                op_ret = -1;
+                op_errno = errno;
+                goto done;
+        }
+
+        iatt_from_stat (&preparent, &lstatbuf);
+        uuid_copy (preparent.ia_gfid, priv->xattrop_vgfid);
+        preparent.ia_ino = -1;
+        uuid_parse (loc->name, gfid);
+        ret = index_del (this, gfid, XATTROP_SUBDIR);
+        if (ret < 0) {
+                op_ret = -1;
+                op_errno = -ret;
+                goto done;
+        }
+        memset (&lstatbuf, 0, sizeof (lstatbuf));
+        ret = lstat (index_dir, &lstatbuf);
+        if (ret < 0) {
+                op_ret = -1;
+                op_errno = errno;
+                goto done;
+        }
+        iatt_from_stat (&postparent, &lstatbuf);
+        uuid_copy (postparent.ia_gfid, priv->xattrop_vgfid);
+        postparent.ia_ino = -1;
+done:
+        INDEX_STACK_UNWIND (unlink, frame, op_ret, op_errno, &preparent,
+                            &postparent, xdata);
+        return 0;
+}
+
+int32_t
+index_getxattr (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, const char *name, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+
+        if (!name)
+                goto out;
+        if (strcmp (GF_XATTROP_INDEX_GFID, name) &&
+            strcmp (GF_BASE_INDICES_HOLDER_GFID, name))
+                goto out;
+
+        stub = fop_getxattr_stub (frame, index_getxattr_wrapper, loc, name,
+                                  xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
+}
+
+int32_t
+index_lookup (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, dict_t *xattr_req)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+
+        if (uuid_compare (loc->gfid, priv->xattrop_vgfid) &&
+            uuid_compare (loc->pargfid, priv->xattrop_vgfid) &&
+            uuid_compare (loc->gfid, priv->base_indices_holder_vgfid) &&
+            uuid_compare (loc->pargfid, priv->base_indices_holder_vgfid))
+                goto normal;
+
+        stub = fop_lookup_stub (frame, index_lookup_wrapper, loc, xattr_req);
+        if (!stub) {
+                STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM, loc->inode,
+                                     NULL, NULL, NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+normal:
+        STACK_WIND (frame, default_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+
+        return 0;
+}
+
+int32_t
+index_readdir (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid) &&
+            uuid_compare (fd->inode->gfid, priv->base_indices_holder_vgfid))
+                goto out;
+
+        if (!uuid_compare (fd->inode->gfid, priv->xattrop_vgfid)) {
+                stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, 
+                                         off, xdata);
+        } else if (!uuid_compare (fd->inode->gfid,
+                                  priv->base_indices_holder_vgfid)) {
+                stub = fop_readdir_stub (frame, base_indices_readdir_wrapper,
+                                         fd, size, off, xdata);
+        }
+
+        if (!stub) {
+                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+        return 0;
+}
+
+int
+index_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        if (uuid_compare (loc->pargfid, priv->xattrop_vgfid))
+                goto out;
+
+        stub = fop_unlink_stub (frame, index_unlink_wrapper, loc, xflag, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL,
+                                     NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_index_mt_end + 1);
+
+        return ret;
+}
+
+int
+init (xlator_t *this)
+{
+        int ret = -1;
+        index_priv_t *priv = NULL;
+        pthread_t thread;
+        pthread_attr_t  w_attr;
+        gf_boolean_t    mutex_inited = _gf_false;
+        gf_boolean_t    cond_inited  = _gf_false;
+        gf_boolean_t    attr_inited  = _gf_false;
+
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"'index' not configured with exactly one child");
+                goto out;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_index_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        LOCK_INIT (&priv->lock);
+        if ((ret = pthread_cond_init(&priv->cond, NULL)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_cond_init failed (%d)", ret);
+                goto out;
+        }
+        cond_inited = _gf_true;
+
+        if ((ret = pthread_mutex_init(&priv->mutex, NULL)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_mutex_init failed (%d)", ret);
+                goto out;
+        }
+        mutex_inited = _gf_true;
+
+        if ((ret = pthread_attr_init (&w_attr)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_attr_init failed (%d)", ret);
+                goto out;
+        }
+        attr_inited = _gf_true;
+
+        ret = pthread_attr_setstacksize (&w_attr, INDEX_THREAD_STACK_SIZE);
+        if (ret == EINVAL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Using default thread stack size");
+        }
+        GF_OPTION_INIT ("index-base", priv->index_basepath, path, out);
+        uuid_generate (priv->index);
+        uuid_generate (priv->xattrop_vgfid);
+        /*base_indices_holder is a directory which contains hard links to
+         * all base indices inside indices/xattrop directory*/
+        uuid_generate (priv->base_indices_holder_vgfid);
+        INIT_LIST_HEAD (&priv->callstubs);
+
+        this->private = priv;
+        ret = gf_thread_create (&thread, &w_attr, index_worker, this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create "
+                        "worker thread, aborting");
+                goto out;
+        }
+        ret = 0;
+out:
+        if (ret) {
+                if (cond_inited)
+                        pthread_cond_destroy (&priv->cond);
+                if (mutex_inited)
+                        pthread_mutex_destroy (&priv->mutex);
+                if (priv)
+                        GF_FREE (priv);
+                this->private = NULL;
+        }
+        if (attr_inited)
+                pthread_attr_destroy (&w_attr);
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        index_priv_t *priv = NULL;
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+        this->private = NULL;
+        LOCK_DESTROY (&priv->lock);
+        pthread_cond_destroy (&priv->cond);
+        pthread_mutex_destroy (&priv->mutex);
+        GF_FREE (priv);
+out:
+        return;
+}
+
+int
+index_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t tmp_cache = 0;
+        if (!inode_ctx_del (inode, this, &tmp_cache))
+                GF_FREE ((index_inode_ctx_t*) (long)tmp_cache);
+
+        return 0;
+}
+
+int32_t
+index_releasedir (xlator_t *this, fd_t *fd)
+{
+        index_fd_ctx_t *fctx = NULL;
+        uint64_t        ctx = 0;
+        int             ret = 0;
+
+        ret = fd_ctx_del (fd, this, &ctx);
+        if (ret < 0)
+                goto out;
+
+        fctx = (index_fd_ctx_t*) (long) ctx;
+        if (fctx->dir)
+                closedir (fctx->dir);
+
+        GF_FREE (fctx);
+out:
+        return 0;
+}
+
+int32_t
+index_release (xlator_t *this, fd_t *fd)
+{
+        index_fd_ctx_t *fctx = NULL;
+        uint64_t        ctx = 0;
+        int             ret = 0;
+
+        ret = fd_ctx_del (fd, this, &ctx);
+        if (ret < 0)
+                goto out;
+
+        fctx = (index_fd_ctx_t*) (long) ctx;
+        GF_FREE (fctx);
+out:
+        return 0;
+}
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+        int     ret = 0;
+        ret = default_notify (this, event, data);
+        return ret;
+}
+
+struct xlator_fops fops = {
+	.xattrop     = index_xattrop,
+	.fxattrop    = index_fxattrop,
+
+        //interface functions follow
+        .getxattr    = index_getxattr,
+        .lookup      = index_lookup,
+        .readdir     = index_readdir,
+        .unlink      = index_unlink
+};
+
+struct xlator_dumpops dumpops;
+
+struct xlator_cbks cbks = {
+        .forget         = index_forget,
+        .release        = index_release,
+        .releasedir     = index_releasedir
+};
+
+struct volume_options options[] = {
+        { .key  = {"index-base" },
+          .type = GF_OPTION_TYPE_PATH,
+          .description = "path where the index files need to be stored",
+        },
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h
new file mode 100644
index 000000000..d6dcb1c23
--- /dev/null
+++ b/xlators/features/index/src/index.h
@@ -0,0 +1,73 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __INDEX_H__
+#define __INDEX_H__
+
+#include "xlator.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "byte-order.h"
+#include "common-utils.h"
+#include "index-mem-types.h"
+
+#define INDEX_THREAD_STACK_SIZE   ((size_t)(1024*1024))
+
+typedef enum {
+        UNKNOWN,
+        IN,
+        NOTIN
+} index_state_t;
+
+typedef struct index_inode_ctx {
+        gf_boolean_t processing;
+        struct list_head callstubs;
+        index_state_t state;
+} index_inode_ctx_t;
+
+typedef struct index_fd_ctx {
+        DIR *dir;
+} index_fd_ctx_t;
+
+typedef enum {
+        sync_not_started,
+        sync_started,
+        synced_state,
+} to_be_healed_states_t;
+
+typedef enum {
+        INDEX_XATTROP,
+        BASE_INDICES_HOLDER,
+} readdir_directory;
+
+typedef struct index_priv {
+        char *index_basepath;
+        uuid_t index;
+        gf_lock_t lock;
+        uuid_t xattrop_vgfid;//virtual gfid of the xattrop index dir
+        uuid_t base_indices_holder_vgfid; //virtual gfid of the
+                                          //to_be_healed_xattrop directory
+        struct list_head callstubs;
+        pthread_mutex_t mutex;
+        pthread_cond_t  cond;
+        to_be_healed_states_t to_be_healed_states;
+} index_priv_t;
+
+#define INDEX_STACK_UNWIND(fop, frame, params ...)      \
+do {                                                    \
+        if (frame) {                                    \
+                inode_t *_inode = frame->local;         \
+                frame->local = NULL;                    \
+                inode_unref (_inode);                   \
+        }                                               \
+        STACK_UNWIND_STRICT (fop, frame, params);       \
+} while (0)
+
+#endif
diff --git a/xlators/features/locks/src/Makefile.am b/xlators/features/locks/src/Makefile.am
index ec4a953eb..0f79731b4 100644
--- a/xlators/features/locks/src/Makefile.am
+++ b/xlators/features/locks/src/Makefile.am
@@ -1,15 +1,18 @@
 xlator_LTLIBRARIES = locks.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-locks_la_LDFLAGS = -module -avoidversion
+locks_la_LDFLAGS = -module -avoid-version
 
-locks_la_SOURCES = common.c posix.c internal.c
-locks_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
+locks_la_SOURCES = common.c posix.c entrylk.c inodelk.c reservelk.c \
+		   clear.c
+locks_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = locks.h common.h
+noinst_HEADERS = locks.h common.h locks-mem-types.h clear.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -fno-strict-aliasing -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src $(GF_CFLAGS) -shared -nostartfiles
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 CLEANFILES = 
 
@@ -17,4 +20,4 @@ uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/posix-locks.so
 
 install-data-hook:
-	ln -sf locks.so $(DESTDIR)$(xlatordir)/posix-locks.so
\ No newline at end of file
+	ln -sf locks.so $(DESTDIR)$(xlatordir)/posix-locks.so
diff --git a/xlators/features/locks/src/clear.c b/xlators/features/locks/src/clear.c
new file mode 100644
index 000000000..124b9ad0f
--- /dev/null
+++ b/xlators/features/locks/src/clear.c
@@ -0,0 +1,424 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <unistd.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <pthread.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "compat.h"
+#include "xlator.h"
+#include "inode.h"
+#include "logging.h"
+#include "common-utils.h"
+
+#include "locks.h"
+#include "common.h"
+#include "statedump.h"
+#include "clear.h"
+
+int
+clrlk_get_kind (char *kind)
+{
+        char  *clrlk_kinds[CLRLK_KIND_MAX] = {"dummy", "blocked", "granted",
+                                              "all"};
+        int   ret_kind                     = CLRLK_KIND_MAX;
+        int   i                            = 0;
+
+        for (i = CLRLK_BLOCKED; i < CLRLK_KIND_MAX; i++) {
+                if (!strcmp (clrlk_kinds[i], kind)) {
+                        ret_kind = i;
+                        break;
+                }
+        }
+
+        return ret_kind;
+}
+
+int
+clrlk_get_type (char *type)
+{
+        char    *clrlk_types[CLRLK_TYPE_MAX]    = {"inode", "entry", "posix"};
+        int     ret_type                        = CLRLK_TYPE_MAX;
+        int     i                               = 0;
+
+        for (i = CLRLK_INODE; i < CLRLK_TYPE_MAX; i++) {
+                if (!strcmp (clrlk_types[i], type)) {
+                        ret_type = i;
+                        break;
+                }
+        }
+
+        return ret_type;
+}
+
+int
+clrlk_get_lock_range (char *range_str, struct gf_flock *ulock,
+                      gf_boolean_t *chk_range)
+{
+        int     ret     =       -1;
+
+        if (!chk_range)
+                goto out;
+
+        if (!range_str) {
+                ret = 0;
+                *chk_range = _gf_false;
+                goto out;
+        }
+
+        if (sscanf (range_str, "%hd,%"PRId64"-""%"PRId64, &ulock->l_whence,
+                    &ulock->l_start, &ulock->l_len) != 3) {
+                goto out;
+        }
+
+        ret = 0;
+        *chk_range = _gf_true;
+out:
+        return ret;
+}
+
+int
+clrlk_parse_args (const char* cmd, clrlk_args *args)
+{
+        char            *opts           = NULL;
+        char            *cur            = NULL;
+        char            *tok            = NULL;
+        char            *sptr           = NULL;
+        char            *free_ptr       = NULL;
+        char            kw[KW_MAX]     = {[KW_TYPE]     = 't',
+                                          [KW_KIND]     = 'k',
+                                          };
+        int             ret             = -1;
+        int             i               = 0;
+
+        GF_ASSERT (cmd);
+        free_ptr = opts = GF_CALLOC (1, strlen (cmd), gf_common_mt_char);
+        if (!opts)
+                goto out;
+
+        if (sscanf (cmd, GF_XATTR_CLRLK_CMD".%s", opts) < 1) {
+                ret = -1;
+                goto out;
+        }
+
+        /*clr_lk_prefix.ttype.kkind.args, args - type specific*/
+        cur = opts;
+        for (i = 0; i < KW_MAX && (tok = strtok_r (cur, ".", &sptr));
+             cur = NULL, i++) {
+                if (tok[0] != kw[i]) {
+                        ret = -1;
+                        goto out;
+                }
+                if (i == KW_TYPE)
+                        args->type = clrlk_get_type (tok+1);
+                if (i == KW_KIND)
+                        args->kind = clrlk_get_kind (tok+1);
+        }
+
+        if ((args->type == CLRLK_TYPE_MAX) || (args->kind == CLRLK_KIND_MAX))
+                goto out;
+
+        /*optional args, neither range nor basename can 'legally' contain
+         * "/" in them*/
+        tok = strtok_r (NULL, "/", &sptr);
+        if (tok)
+                args->opts = gf_strdup (tok);
+
+        ret = 0;
+out:
+        GF_FREE (free_ptr);
+        return ret;
+}
+
+int
+clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
+                     int *blkd, int *granted, int *op_errno)
+{
+        posix_lock_t            *plock          = NULL;
+        posix_lock_t            *tmp            = NULL;
+        struct gf_flock         ulock           = {0, };
+        int                     ret             = -1;
+        int                     bcount          = 0;
+        int                     gcount          = 0;
+        gf_boolean_t            chk_range       = _gf_false;
+
+        if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
+                *op_errno = EINVAL;
+                goto out;
+        }
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (plock, tmp, &pl_inode->ext_list,
+                                          list) {
+                        if ((plock->blocked &&
+                             !(args->kind & CLRLK_BLOCKED)) ||
+                            (!plock->blocked &&
+                             !(args->kind & CLRLK_GRANTED)))
+                                continue;
+
+                        if (chk_range &&
+                            (plock->user_flock.l_whence != ulock.l_whence
+                            || plock->user_flock.l_start != ulock.l_start
+                            || plock->user_flock.l_len != ulock.l_len))
+                                continue;
+
+                        list_del_init (&plock->list);
+                        if (plock->blocked) {
+                                bcount++;
+                                pl_trace_out (this, plock->frame, NULL, NULL,
+                                              F_SETLKW, &plock->user_flock,
+                                              -1, EAGAIN, NULL);
+
+                                STACK_UNWIND_STRICT (lk, plock->frame, -1, EAGAIN,
+                                                     &plock->user_flock, NULL);
+
+                        } else {
+                                gcount++;
+                        }
+                        GF_FREE (plock);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+        grant_blocked_locks (this, pl_inode);
+        ret = 0;
+out:
+        *blkd    = bcount;
+        *granted = gcount;
+        return ret;
+}
+
+/* Returns 0 on success and -1 on failure */
+int
+clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                     clrlk_args *args, int *blkd, int *granted, int *op_errno)
+{
+        pl_inode_lock_t         *ilock          = NULL;
+        pl_inode_lock_t         *tmp            = NULL;
+        struct gf_flock         ulock           = {0, };
+        int                     ret             = -1;
+        int                     bcount          = 0;
+        int                     gcount          = 0;
+        gf_boolean_t            chk_range       = _gf_false;
+        struct list_head        released;
+
+        INIT_LIST_HEAD (&released);
+        if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
+                *op_errno = EINVAL;
+                goto out;
+        }
+
+        if (args->kind & CLRLK_BLOCKED)
+                goto blkd;
+
+        if (args->kind & CLRLK_GRANTED)
+                goto granted;
+
+blkd:
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (ilock, tmp, &dom->blocked_inodelks,
+                                          blocked_locks) {
+                        if (chk_range &&
+                            (ilock->user_flock.l_whence != ulock.l_whence
+                            || ilock->user_flock.l_start != ulock.l_start
+                            || ilock->user_flock.l_len != ulock.l_len))
+                                continue;
+
+                        bcount++;
+                        list_del_init (&ilock->blocked_locks);
+                        list_add (&ilock->blocked_locks, &released);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (ilock, tmp, &released, blocked_locks) {
+                list_del_init (&ilock->blocked_locks);
+                pl_trace_out (this, ilock->frame, NULL, NULL, F_SETLKW,
+                              &ilock->user_flock, -1, EAGAIN,
+                              ilock->volume);
+                STACK_UNWIND_STRICT (inodelk, ilock->frame, -1,
+                                     EAGAIN, NULL);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (ilock);
+        }
+
+        if (!(args->kind & CLRLK_GRANTED)) {
+                ret = 0;
+                goto out;
+        }
+
+granted:
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (ilock, tmp, &dom->inodelk_list,
+                                          list) {
+                        if (chk_range &&
+                            (ilock->user_flock.l_whence != ulock.l_whence
+                            || ilock->user_flock.l_start != ulock.l_start
+                            || ilock->user_flock.l_len != ulock.l_len))
+                                continue;
+
+                        gcount++;
+                        list_del_init (&ilock->list);
+                        list_add (&ilock->list, &released);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (ilock, tmp, &released, list) {
+                list_del_init (&ilock->list);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (ilock);
+        }
+
+        ret = 0;
+out:
+        grant_blocked_inode_locks (this, pl_inode, dom);
+        *blkd    = bcount;
+        *granted = gcount;
+        return ret;
+}
+
+/* Returns 0 on success and -1 on failure */
+int
+clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                     clrlk_args *args, int *blkd, int *granted, int *op_errno)
+{
+        pl_entry_lock_t         *elock          = NULL;
+        pl_entry_lock_t         *tmp            = NULL;
+        int                     bcount          = 0;
+        int                     gcount          = 0;
+        int                     ret             = -1;
+        struct list_head        removed;
+        struct list_head        released;
+
+        INIT_LIST_HEAD (&released);
+        if (args->kind & CLRLK_BLOCKED)
+                goto blkd;
+
+        if (args->kind & CLRLK_GRANTED)
+                goto granted;
+
+blkd:
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (elock, tmp, &dom->blocked_entrylks,
+                                          blocked_locks) {
+                        if (args->opts) {
+                                if (!elock->basename ||
+                                    strcmp (elock->basename, args->opts))
+                                        continue;
+                        }
+
+                        bcount++;
+
+                        list_del_init (&elock->blocked_locks);
+                        list_add_tail (&elock->blocked_locks, &released);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (elock, tmp, &released, blocked_locks) {
+                list_del_init (&elock->blocked_locks);
+                entrylk_trace_out (this, elock->frame, elock->volume, NULL, NULL,
+                                   elock->basename, ENTRYLK_LOCK, elock->type,
+                                   -1, EAGAIN);
+                STACK_UNWIND_STRICT (entrylk, elock->frame, -1, EAGAIN, NULL);
+                GF_FREE ((char *) elock->basename);
+                GF_FREE (elock->connection_id);
+                GF_FREE (elock);
+        }
+
+        if (!(args->kind & CLRLK_GRANTED)) {
+                ret = 0;
+                goto out;
+        }
+
+granted:
+        INIT_LIST_HEAD (&removed);
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (elock, tmp, &dom->entrylk_list,
+                                          domain_list) {
+                        if (args->opts) {
+                                if (!elock->basename ||
+                                    strcmp (elock->basename, args->opts))
+                                        continue;
+                        }
+
+                        gcount++;
+                        list_del_init (&elock->domain_list);
+                        list_add_tail (&elock->domain_list, &removed);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (elock, tmp, &removed, domain_list) {
+                grant_blocked_entry_locks (this, pl_inode, elock, dom);
+        }
+
+        ret = 0;
+out:
+        *blkd    = bcount;
+        *granted = gcount;
+        return ret;
+}
+
+int
+clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
+                                clrlk_args *args, int *blkd, int *granted,
+                                int *op_errno)
+{
+        pl_dom_list_t   *dom            = NULL;
+        int             ret             = -1;
+        int             tmp_bcount      = 0;
+        int             tmp_gcount      = 0;
+
+        if (list_empty (&pl_inode->dom_list)) {
+                ret = 0;
+                goto out;
+        }
+
+        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+                tmp_bcount = tmp_gcount = 0;
+
+                switch (args->type)
+                {
+                case CLRLK_INODE:
+                        ret = clrlk_clear_inodelk (this, pl_inode, dom, args,
+                                             &tmp_bcount, &tmp_gcount,
+                                             op_errno);
+                        if (ret)
+                                goto out;
+                        break;
+                case CLRLK_ENTRY:
+                        ret = clrlk_clear_entrylk (this, pl_inode, dom, args,
+                                             &tmp_bcount, &tmp_gcount,
+                                             op_errno);
+                        if (ret)
+                                goto out;
+                        break;
+                }
+
+                *blkd    += tmp_bcount;
+                *granted += tmp_gcount;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/features/locks/src/clear.h b/xlators/features/locks/src/clear.h
new file mode 100644
index 000000000..511f3f74a
--- /dev/null
+++ b/xlators/features/locks/src/clear.h
@@ -0,0 +1,76 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __CLEAR_H__
+#define __CLEAR_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "compat-errno.h"
+#include "stack.h"
+#include "call-stub.h"
+#include "locks.h"
+
+typedef enum {
+        CLRLK_INODE,
+        CLRLK_ENTRY,
+        CLRLK_POSIX,
+        CLRLK_TYPE_MAX
+} clrlk_type;
+
+typedef enum {
+        CLRLK_BLOCKED = 1,
+        CLRLK_GRANTED,
+        CLRLK_ALL,
+        CLRLK_KIND_MAX
+} clrlk_kind;
+
+typedef enum {
+        KW_TYPE,
+        KW_KIND,
+        /*add new keywords here*/
+        KW_MAX
+} clrlk_opts;
+
+struct _clrlk_args;
+typedef struct _clrlk_args clrlk_args;
+
+struct _clrlk_args {
+        int type;
+        int kind;
+        char *opts;
+};
+
+int
+clrlk_get__kind (char *kind);
+int
+clrlk_get_type (char *type);
+int
+clrlk_get_lock_range (char *range_str, struct gf_flock *ulock,
+                      gf_boolean_t *chk_range);
+int
+clrlk_parse_args (const char* cmd, clrlk_args *args);
+
+int
+clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
+                     int *blkd, int *granted, int *op_errno);
+int
+clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                     clrlk_args *args, int *blkd, int *granted, int *op_errno);
+int
+clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                     clrlk_args *args, int *blkd, int *granted, int *op_errno);
+int
+clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
+                                clrlk_args *args, int *blkd, int *granted,
+                                int *op_errno);
+#endif /* __CLEAR_H__ */
diff --git a/xlators/features/locks/src/common.c b/xlators/features/locks/src/common.c
index 9ac1250cc..b3309580d 100644
--- a/xlators/features/locks/src/common.c
+++ b/xlators/features/locks/src/common.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -35,84 +25,477 @@
 #include "common-utils.h"
 
 #include "locks.h"
+#include "common.h"
 
 
-int
-pl_is_lock_grantable (pl_inode_t *pl_inode, posix_lock_t *lock,
-		      gf_lk_domain_t dom);
+static int
+__is_lock_grantable (pl_inode_t *pl_inode, posix_lock_t *lock);
 static void
-__insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock,
-		    gf_lk_domain_t dom);
+__insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock);
+static int
+pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
+                        posix_lock_t *old_lock);
+
+static pl_dom_list_t *
+__allocate_domain (const char *volume)
+{
+        pl_dom_list_t *dom = NULL;
+
+        dom = GF_CALLOC (1, sizeof (*dom),
+                         gf_locks_mt_pl_dom_list_t);
+        if (!dom)
+                goto out;
+
+        dom->domain = gf_strdup(volume);
+        if (!dom->domain)
+                goto out;
+
+        gf_log ("posix-locks", GF_LOG_TRACE,
+                "New domain allocated: %s", dom->domain);
+
+        INIT_LIST_HEAD (&dom->inode_list);
+        INIT_LIST_HEAD (&dom->entrylk_list);
+        INIT_LIST_HEAD (&dom->blocked_entrylks);
+        INIT_LIST_HEAD (&dom->inodelk_list);
+        INIT_LIST_HEAD (&dom->blocked_inodelks);
+
+out:
+        if (dom && (NULL == dom->domain)) {
+                GF_FREE (dom);
+                dom = NULL;
+        }
+
+        return dom;
+}
+
+/* Returns domain for the lock. If domain is not present,
+ * allocates a domain and returns it
+ */
+pl_dom_list_t *
+get_domain (pl_inode_t *pl_inode, const char *volume)
+{
+        pl_dom_list_t *dom = NULL;
+
+        GF_VALIDATE_OR_GOTO ("posix-locks", pl_inode, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", volume, out);
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+                        if (strcmp (dom->domain, volume) == 0)
+                                goto unlock;
+                }
+
+                dom = __allocate_domain (volume);
+                if (dom)
+                        list_add (&dom->inode_list, &pl_inode->dom_list);
+        }
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
+        if (dom) {
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s found", volume);
+        } else {
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s not found", volume);
+        }
+out:
+        return dom;
+}
+
+unsigned long
+fd_to_fdnum (fd_t *fd)
+{
+        return ((unsigned long) fd);
+}
+
+fd_t *
+fd_from_fdnum (posix_lock_t *lock)
+{
+        return ((fd_t *) lock->fd_num);
+}
+
+int
+__pl_inode_is_empty (pl_inode_t *pl_inode)
+{
+        pl_dom_list_t *dom = NULL;
+        int            is_empty = 1;
+
+        if (!list_empty (&pl_inode->ext_list))
+                is_empty = 0;
+
+        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+                if (!list_empty (&dom->entrylk_list))
+                        is_empty = 0;
+
+                if (!list_empty (&dom->inodelk_list))
+                        is_empty = 0;
+        }
+
+        return is_empty;
+}
+
+void
+pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame)
+{
+        snprintf (str, size, "Pid=%llu, lk-owner=%s, Client=%p, Frame=%llu",
+                  (unsigned long long) frame->root->pid,
+                  lkowner_utoa (&frame->root->lk_owner),
+                  frame->root->client,
+                  (unsigned long long) frame->root->unique);
+}
+
+
+void
+pl_print_lockee (char *str, int size, fd_t *fd, loc_t *loc)
+{
+        inode_t  *inode = NULL;
+        char     *ipath = NULL;
+        int       ret = 0;
+
+        if (fd)
+                inode = fd->inode;
+        if (loc)
+                inode = loc->inode;
+
+        if (!inode) {
+                snprintf (str, size, "<nul>");
+                return;
+        }
+
+        if (loc && loc->path) {
+                ipath = gf_strdup (loc->path);
+        } else {
+                ret = inode_path (inode, NULL, &ipath);
+                if (ret <= 0)
+                        ipath = NULL;
+        }
+
+        snprintf (str, size, "gfid=%s, fd=%p, path=%s",
+                  uuid_utoa (inode->gfid), fd,
+                  ipath ? ipath : "<nul>");
+
+        GF_FREE (ipath);
+}
+
+
+void
+pl_print_lock (char *str, int size, int cmd,
+               struct gf_flock *flock, gf_lkowner_t *owner)
+{
+        char *cmd_str = NULL;
+        char *type_str = NULL;
+
+        switch (cmd) {
+#if F_GETLK != F_GETLK64
+        case F_GETLK64:
+#endif
+        case F_GETLK:
+                cmd_str = "GETLK";
+                break;
+
+#if F_SETLK != F_SETLK64
+        case F_SETLK64:
+#endif
+        case F_SETLK:
+                cmd_str = "SETLK";
+                break;
+
+#if F_SETLKW != F_SETLKW64
+        case F_SETLKW64:
+#endif
+        case F_SETLKW:
+                cmd_str = "SETLKW";
+                break;
+
+        default:
+                cmd_str = "UNKNOWN";
+                break;
+        }
+
+        switch (flock->l_type) {
+        case F_RDLCK:
+                type_str = "READ";
+                break;
+        case F_WRLCK:
+                type_str = "WRITE";
+                break;
+        case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+        default:
+                type_str = "UNKNOWN";
+                break;
+        }
+
+        snprintf (str, size, "lock=FCNTL, cmd=%s, type=%s, "
+                  "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
+                  cmd_str, type_str, (unsigned long long) flock->l_start,
+                  (unsigned long long) flock->l_len,
+                  (unsigned long long) flock->l_pid,
+                  lkowner_utoa (owner));
+}
+
+
+void
+pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+             int cmd, struct gf_flock *flock, const char *domain)
+{
+        posix_locks_private_t  *priv = NULL;
+        char                    pl_locker[256];
+        char                    pl_lockee[256];
+        char                    pl_lock[256];
+
+        priv = this->private;
+
+        if (!priv->trace)
+                return;
+
+        pl_print_locker (pl_locker, 256, this, frame);
+        pl_print_lockee (pl_lockee, 256, fd, loc);
+        if (domain)
+                pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
+        else
+                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}",
+                pl_locker, pl_lockee, pl_lock);
+}
+
+
+void
+pl_print_verdict (char *str, int size, int op_ret, int op_errno)
+{
+        char *verdict = NULL;
+
+        if (op_ret == 0) {
+                verdict = "GRANTED";
+        } else {
+                switch (op_errno) {
+                case EAGAIN:
+                        verdict = "TRYAGAIN";
+                        break;
+                default:
+                        verdict = strerror (op_errno);
+                }
+        }
+
+        snprintf (str, size, "%s", verdict);
+}
+
+
+void
+pl_trace_out (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+              int cmd, struct gf_flock *flock, int op_ret, int op_errno, const char *domain)
+
+{
+        posix_locks_private_t  *priv = NULL;
+        char                    pl_locker[256];
+        char                    pl_lockee[256];
+        char                    pl_lock[256];
+        char                    verdict[32];
+
+        priv = this->private;
+
+        if (!priv->trace)
+                return;
+
+        pl_print_locker (pl_locker, 256, this, frame);
+        pl_print_lockee (pl_lockee, 256, fd, loc);
+        if (domain)
+                pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
+        else
+                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
+
+        pl_print_verdict (verdict, 32, op_ret, op_errno);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}",
+                verdict, pl_locker, pl_lockee, pl_lock);
+}
+
+
+void
+pl_trace_block (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+                int cmd, struct gf_flock *flock, const char *domain)
+
+{
+        posix_locks_private_t  *priv = NULL;
+        char                    pl_locker[256];
+        char                    pl_lockee[256];
+        char                    pl_lock[256];
+
+        priv = this->private;
+
+        if (!priv->trace)
+                return;
+
+        pl_print_locker (pl_locker, 256, this, frame);
+        pl_print_lockee (pl_lockee, 256, fd, loc);
+        if (domain)
+                pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
+        else
+                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}",
+                pl_locker, pl_lockee, pl_lock);
+}
+
+
+void
+pl_trace_flush (xlator_t *this, call_frame_t *frame, fd_t *fd)
+{
+        posix_locks_private_t  *priv = NULL;
+        char                    pl_locker[256];
+        char                    pl_lockee[256];
+        pl_inode_t             *pl_inode = NULL;
+
+        priv = this->private;
+
+        if (!priv->trace)
+                return;
+
+        pl_inode = pl_inode_get (this, fd->inode);
+
+        if (pl_inode && __pl_inode_is_empty (pl_inode))
+                return;
+
+        pl_print_locker (pl_locker, 256, this, frame);
+        pl_print_lockee (pl_lockee, 256, fd, NULL);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[FLUSH] Locker = {%s} Lockee = {%s}",
+                pl_locker, pl_lockee);
+}
+
+void
+pl_trace_release (xlator_t *this, fd_t *fd)
+{
+        posix_locks_private_t  *priv = NULL;
+        char                    pl_lockee[256];
+
+        priv = this->private;
+
+        if (!priv->trace)
+                return;
+
+        pl_print_lockee (pl_lockee, 256, fd, NULL);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[RELEASE] Lockee = {%s}", pl_lockee);
+}
+
+
+void
+pl_update_refkeeper (xlator_t *this, inode_t *inode)
+{
+        pl_inode_t *pl_inode  = NULL;
+        int         is_empty  = 0;
+        int         need_unref = 0;
+        int         need_ref = 0;
+
+        pl_inode = pl_inode_get (this, inode);
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                is_empty = __pl_inode_is_empty (pl_inode);
+
+                if (is_empty && pl_inode->refkeeper) {
+                        need_unref = 1;
+                        pl_inode->refkeeper = NULL;
+                }
+
+                if (!is_empty && !pl_inode->refkeeper) {
+                        need_ref = 1;
+                        pl_inode->refkeeper = inode;
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        if (need_unref)
+                inode_unref (inode);
+
+        if (need_ref)
+                inode_ref (inode);
+}
 
 
 pl_inode_t *
 pl_inode_get (xlator_t *this, inode_t *inode)
 {
-	pl_inode_t *pl_inode     = NULL;
-	mode_t      st_mode      = 0;
-	uint64_t    tmp_pl_inode = 0;
-	int         ret          = 0;
-	
-	LOCK (&inode->lock);
-	{
-		ret = inode_ctx_get (inode, this, &tmp_pl_inode);
-		if (ret == 0) {
-			pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-			goto out;
-		}
-
-		pl_inode = CALLOC (1, sizeof (*pl_inode));
-		if (!pl_inode) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"out of memory :(");
-			goto out;
-		}
-
-		st_mode  = inode->st_mode;
-		if ((st_mode & S_ISGID) && !(st_mode & S_IXGRP))
-			pl_inode->mandatory = 1;
-
-
-		pthread_mutex_init (&pl_inode->mutex, NULL);
-		
-		INIT_LIST_HEAD (&pl_inode->dir_list);
-		INIT_LIST_HEAD (&pl_inode->ext_list);
-		INIT_LIST_HEAD (&pl_inode->int_list);
-		INIT_LIST_HEAD (&pl_inode->rw_list);
-
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)pl_inode);
-	}
-out:
-	UNLOCK (&inode->lock);
-	return pl_inode;
+        uint64_t    tmp_pl_inode = 0;
+        pl_inode_t *pl_inode = NULL;
+        int         ret = 0;
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
+                if (ret == 0) {
+                        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+                        goto unlock;
+                }
+                pl_inode = GF_CALLOC (1, sizeof (*pl_inode),
+                                      gf_locks_mt_pl_inode_t);
+                if (!pl_inode) {
+                        goto unlock;
+                }
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Allocating new pl inode");
+
+                pthread_mutex_init (&pl_inode->mutex, NULL);
+
+                INIT_LIST_HEAD (&pl_inode->dom_list);
+                INIT_LIST_HEAD (&pl_inode->ext_list);
+                INIT_LIST_HEAD (&pl_inode->rw_list);
+                INIT_LIST_HEAD (&pl_inode->reservelk_list);
+                INIT_LIST_HEAD (&pl_inode->blocked_reservelks);
+                INIT_LIST_HEAD (&pl_inode->blocked_calls);
+
+                __inode_ctx_put (inode, this, (uint64_t)(long)(pl_inode));
+        }
+unlock:
+        UNLOCK (&inode->lock);
+
+        return pl_inode;
 }
 
 
 /* Create a new posix_lock_t */
 posix_lock_t *
-new_posix_lock (struct flock *flock, transport_t *transport, pid_t client_pid)
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
+                gf_lkowner_t *owner, fd_t *fd)
 {
-	posix_lock_t *lock = NULL;
+        posix_lock_t *lock = NULL;
+
+        GF_VALIDATE_OR_GOTO ("posix-locks", flock, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", client, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", fd, out);
 
-	lock = CALLOC (1, sizeof (posix_lock_t));
-	if (!lock) {
-		return NULL;
-	}
+        lock = GF_CALLOC (1, sizeof (posix_lock_t),
+                          gf_locks_mt_posix_lock_t);
+        if (!lock) {
+                goto out;
+        }
 
-	lock->fl_start = flock->l_start;
-	lock->fl_type  = flock->l_type;
+        lock->fl_start = flock->l_start;
+        lock->fl_type  = flock->l_type;
 
-	if (flock->l_len == 0)
-		lock->fl_end = LLONG_MAX;
-	else
-		lock->fl_end = flock->l_start + flock->l_len - 1;
+        if (flock->l_len == 0)
+                lock->fl_end = LLONG_MAX;
+        else
+                lock->fl_end = flock->l_start + flock->l_len - 1;
 
-	lock->transport  = transport;
-	lock->client_pid = client_pid;
+        lock->client     = client;
+        lock->fd_num     = fd_to_fdnum (fd);
+        lock->fd         = fd;
+        lock->client_pid = client_pid;
+        lock->owner      = *owner;
 
-	INIT_LIST_HEAD (&lock->list);
+        INIT_LIST_HEAD (&lock->list);
 
-	return lock;
+out:
+        return lock;
 }
 
 
@@ -120,7 +503,7 @@ new_posix_lock (struct flock *flock, transport_t *transport, pid_t client_pid)
 void
 __delete_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-	list_del_init (&lock->list);
+        list_del_init (&lock->list);
 }
 
 
@@ -128,32 +511,37 @@ __delete_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
 void
 __destroy_lock (posix_lock_t *lock)
 {
-	free (lock);
+        GF_FREE (lock);
 }
 
 
-/* Convert a posix_lock to a struct flock */
+/* Convert a posix_lock to a struct gf_flock */
 void
-posix_lock_to_flock (posix_lock_t *lock, struct flock *flock)
+posix_lock_to_flock (posix_lock_t *lock, struct gf_flock *flock)
 {
-	flock->l_pid   = lock->client_pid;
-	flock->l_type  = lock->fl_type;
-	flock->l_start = lock->fl_start;
-
-	if (lock->fl_end == 0)
-		flock->l_len = LLONG_MAX;
-	else
-		flock->l_len = lock->fl_end - lock->fl_start + 1;
+        flock->l_pid   = lock->client_pid;
+        flock->l_type  = lock->fl_type;
+        flock->l_start = lock->fl_start;
+        flock->l_owner = lock->owner;
+
+        if (lock->fl_end == LLONG_MAX)
+                flock->l_len = 0;
+        else
+                flock->l_len = lock->fl_end - lock->fl_start + 1;
 }
 
-
 /* Insert the lock into the inode's lock list */
-void
-pl_insert_lock (pl_inode_t *pl_inode, posix_lock_t *lock, gf_lk_domain_t dom)
+static void
+__insert_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-	list_add_tail (&lock->list, DOMAIN_HEAD (pl_inode, dom));
+        if (lock->blocked)
+                gettimeofday (&lock->blkd_time, NULL);
+        else
+                gettimeofday (&lock->granted_time, NULL);
 
-	return;
+        list_add_tail (&lock->list, &pl_inode->ext_list);
+
+        return;
 }
 
 
@@ -161,14 +549,14 @@ pl_insert_lock (pl_inode_t *pl_inode, posix_lock_t *lock, gf_lk_domain_t dom)
 int
 locks_overlap (posix_lock_t *l1, posix_lock_t *l2)
 {
-	/* 
-	   Note:
-	   FUSE always gives us absolute offsets, so no need to worry 
-	   about SEEK_CUR or SEEK_END
-	*/
-
-	return ((l1->fl_end >= l2->fl_start) &&
-		(l2->fl_end >= l1->fl_start));
+        /*
+           Note:
+           FUSE always gives us absolute offsets, so no need to worry
+           about SEEK_CUR or SEEK_END
+        */
+
+        return ((l1->fl_end >= l2->fl_start) &&
+                (l2->fl_end >= l1->fl_start));
 }
 
 
@@ -176,24 +564,26 @@ locks_overlap (posix_lock_t *l1, posix_lock_t *l2)
 int
 same_owner (posix_lock_t *l1, posix_lock_t *l2)
 {
-	return ((l1->client_pid == l2->client_pid) &&
-		(l1->transport  == l2->transport));
+
+        return (is_same_lkowner (&l1->owner, &l2->owner) &&
+                (l1->client == l2->client));
+
 }
 
 
 /* Delete all F_UNLCK locks */
 void
-__delete_unlck_locks (pl_inode_t *pl_inode, gf_lk_domain_t dom)
+__delete_unlck_locks (pl_inode_t *pl_inode)
 {
-	posix_lock_t *l = NULL;
-	posix_lock_t *tmp = NULL;
-
-	list_for_each_entry_safe (l, tmp, DOMAIN_HEAD (pl_inode, dom), list) {
-		if (l->fl_type == F_UNLCK) {
-			__delete_lock (pl_inode, l);
-			__destroy_lock (l);
-		}
-	}
+        posix_lock_t *l = NULL;
+        posix_lock_t *tmp = NULL;
+
+        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
+                if (l->fl_type == F_UNLCK) {
+                        __delete_lock (pl_inode, l);
+                        __destroy_lock (l);
+                }
+        }
 }
 
 
@@ -201,129 +591,202 @@ __delete_unlck_locks (pl_inode_t *pl_inode, gf_lk_domain_t dom)
 static posix_lock_t *
 add_locks (posix_lock_t *l1, posix_lock_t *l2)
 {
-	posix_lock_t *sum = NULL;
+        posix_lock_t *sum = NULL;
 
-	sum = CALLOC (1, sizeof (posix_lock_t));
-	if (!sum)
-		return NULL;
+        sum = GF_CALLOC (1, sizeof (posix_lock_t),
+                         gf_locks_mt_posix_lock_t);
+        if (!sum)
+                return NULL;
 
-	sum->fl_start = min (l1->fl_start, l2->fl_start);
-	sum->fl_end   = max (l1->fl_end, l2->fl_end);
+        sum->fl_start = min (l1->fl_start, l2->fl_start);
+        sum->fl_end   = max (l1->fl_end, l2->fl_end);
 
-	return sum;
+        return sum;
 }
 
 /* Subtract two locks */
 struct _values {
-	posix_lock_t *locks[3];
+        posix_lock_t *locks[3];
 };
 
 /* {big} must always be contained inside {small} */
 static struct _values
 subtract_locks (posix_lock_t *big, posix_lock_t *small)
 {
-	struct _values v = { .locks = {0, 0, 0} };
-  
-	if ((big->fl_start == small->fl_start) && 
-	    (big->fl_end   == small->fl_end)) {  
-		/* both edges coincide with big */
-		v.locks[0] = CALLOC (1, sizeof (posix_lock_t));
-		ERR_ABORT (v.locks[0]);
-		memcpy (v.locks[0], big, sizeof (posix_lock_t));
-		v.locks[0]->fl_type = small->fl_type;
-	}
-	else if ((small->fl_start > big->fl_start) &&
-		 (small->fl_end   < big->fl_end)) {
-		/* both edges lie inside big */
-		v.locks[0] = CALLOC (1, sizeof (posix_lock_t));
-		ERR_ABORT (v.locks[0]);
-		v.locks[1] = CALLOC (1, sizeof (posix_lock_t));
-		ERR_ABORT (v.locks[1]);
-		v.locks[2] = CALLOC (1, sizeof (posix_lock_t));
-		ERR_ABORT (v.locks[2]);
-
-		memcpy (v.locks[0], big, sizeof (posix_lock_t));
-		v.locks[0]->fl_end = small->fl_start - 1;
-
-		memcpy (v.locks[1], small, sizeof (posix_lock_t));
-		memcpy (v.locks[2], big, sizeof (posix_lock_t));
-		v.locks[2]->fl_start = small->fl_end + 1;
-	}
-	/* one edge coincides with big */
-	else if (small->fl_start == big->fl_start) {
-		v.locks[0] = CALLOC (1, sizeof (posix_lock_t));
-		ERR_ABORT (v.locks[0]);
-		v.locks[1] = CALLOC (1, sizeof (posix_lock_t));
-		ERR_ABORT (v.locks[1]);
-    
-		memcpy (v.locks[0], big, sizeof (posix_lock_t));
-		v.locks[0]->fl_start = small->fl_end + 1;
-    
-		memcpy (v.locks[1], small, sizeof (posix_lock_t));
-	}
-	else if (small->fl_end   == big->fl_end) {
-		v.locks[0] = CALLOC (1, sizeof (posix_lock_t));
-		ERR_ABORT (v.locks[0]);
-		v.locks[1] = CALLOC (1, sizeof (posix_lock_t));
-		ERR_ABORT (v.locks[1]);
-
-		memcpy (v.locks[0], big, sizeof (posix_lock_t));
-		v.locks[0]->fl_end = small->fl_start - 1;
-    
-		memcpy (v.locks[1], small, sizeof (posix_lock_t));
-	}
-	else {
-		gf_log ("posix-locks", GF_LOG_DEBUG, 
-			"unexpected case in subtract_locks");
-	}
-
-	return v;
-}
-
-/* 
-   Start searching from {begin}, and return the first lock that
-   conflicts, NULL if no conflict
-   If {begin} is NULL, then start from the beginning of the list
+
+        struct _values v = { .locks = {0, 0, 0} };
+
+        if ((big->fl_start == small->fl_start) &&
+            (big->fl_end   == small->fl_end)) {
+                /* both edges coincide with big */
+                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
+                                        gf_locks_mt_posix_lock_t);
+                if (!v.locks[0])
+                        goto out;
+                memcpy (v.locks[0], big, sizeof (posix_lock_t));
+                v.locks[0]->fl_type = small->fl_type;
+                goto done;
+        }
+
+        if ((small->fl_start > big->fl_start) &&
+            (small->fl_end   < big->fl_end)) {
+                /* both edges lie inside big */
+                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
+                                        gf_locks_mt_posix_lock_t);
+                if (!v.locks[0])
+                        goto out;
+
+                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
+                                        gf_locks_mt_posix_lock_t);
+                if (!v.locks[1])
+                        goto out;
+
+                v.locks[2] = GF_CALLOC (1, sizeof (posix_lock_t),
+                                        gf_locks_mt_posix_lock_t);
+                if (!v.locks[1])
+                        goto out;
+
+                memcpy (v.locks[0], big, sizeof (posix_lock_t));
+                v.locks[0]->fl_end = small->fl_start - 1;
+
+                memcpy (v.locks[1], small, sizeof (posix_lock_t));
+
+                memcpy (v.locks[2], big, sizeof (posix_lock_t));
+                v.locks[2]->fl_start = small->fl_end + 1;
+                goto done;
+
+        }
+
+        /* one edge coincides with big */
+        if (small->fl_start == big->fl_start) {
+                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
+                                        gf_locks_mt_posix_lock_t);
+                if (!v.locks[0])
+                        goto out;
+
+                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
+                                        gf_locks_mt_posix_lock_t);
+                if (!v.locks[1])
+                        goto out;
+
+                memcpy (v.locks[0], big, sizeof (posix_lock_t));
+                v.locks[0]->fl_start = small->fl_end + 1;
+
+                memcpy (v.locks[1], small, sizeof (posix_lock_t));
+                goto done;
+        }
+
+        if (small->fl_end  == big->fl_end) {
+                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
+                                        gf_locks_mt_posix_lock_t);
+                if (!v.locks[0])
+                        goto out;
+
+                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
+                                        gf_locks_mt_posix_lock_t);
+                if (!v.locks[1])
+                        goto out;
+
+                memcpy (v.locks[0], big, sizeof (posix_lock_t));
+                v.locks[0]->fl_end = small->fl_start - 1;
+
+                memcpy (v.locks[1], small, sizeof (posix_lock_t));
+                goto done;
+        }
+
+        GF_ASSERT (0);
+        gf_log ("posix-locks", GF_LOG_ERROR, "Unexpected case in subtract_locks");
+
+out:
+        if (v.locks[0]) {
+                GF_FREE (v.locks[0]);
+                v.locks[0] = NULL;
+        }
+        if (v.locks[1]) {
+                GF_FREE (v.locks[1]);
+                v.locks[1] = NULL;
+        }
+        if (v.locks[2]) {
+                GF_FREE (v.locks[2]);
+                v.locks[2] = NULL;
+        }
+
+done:
+        return v;
+}
+
+static posix_lock_t *
+first_conflicting_overlap (pl_inode_t *pl_inode, posix_lock_t *lock)
+{
+        posix_lock_t *l = NULL;
+        posix_lock_t *conf = NULL;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (l, &pl_inode->ext_list, list) {
+                        if (l->blocked)
+                                continue;
+
+                        if (locks_overlap (l, lock)) {
+                                if (same_owner (l, lock))
+                                        continue;
+
+                                if ((l->fl_type == F_WRLCK) ||
+                                    (lock->fl_type == F_WRLCK)) {
+                                        conf = l;
+                                        goto unlock;
+                                }
+                        }
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        return conf;
+}
+
+/*
+  Start searching from {begin}, and return the first lock that
+  conflicts, NULL if no conflict
+  If {begin} is NULL, then start from the beginning of the list
 */
 static posix_lock_t *
-first_overlap (pl_inode_t *pl_inode, posix_lock_t *lock,
-	       gf_lk_domain_t dom)
+first_overlap (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-	posix_lock_t *l = NULL;
+        posix_lock_t *l = NULL;
 
-	list_for_each_entry (l, DOMAIN_HEAD (pl_inode, dom), list) {
-		if (l->blocked)
-			continue;
+        list_for_each_entry (l, &pl_inode->ext_list, list) {
+                if (l->blocked)
+                        continue;
 
-		if (locks_overlap (l, lock))
-			return l;
-	}
+                if (locks_overlap (l, lock))
+                        return l;
+        }
 
-	return NULL;
+        return NULL;
 }
 
 
 
 /* Return true if lock is grantable */
-int
-pl_is_lock_grantable (pl_inode_t *pl_inode, posix_lock_t *lock,
-		      gf_lk_domain_t dom)
+static int
+__is_lock_grantable (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-	posix_lock_t *l = NULL;
-	int           ret = 1;
-
-	list_for_each_entry (l, DOMAIN_HEAD (pl_inode, dom), list) {
-		if (!l->blocked && locks_overlap (lock, l)) {
-			if (((l->fl_type == F_WRLCK)
-			     || (lock->fl_type == F_WRLCK))
-			    && (lock->fl_type != F_UNLCK)
-			    && !same_owner (l, lock)) {
-				ret = 0;
-				break;
-			}
-		}
-	}
-	return ret;
+        posix_lock_t *l = NULL;
+        int           ret = 1;
+
+        list_for_each_entry (l, &pl_inode->ext_list, list) {
+                if (!l->blocked && locks_overlap (lock, l)) {
+                        if (((l->fl_type == F_WRLCK)
+                             || (lock->fl_type == F_WRLCK))
+                            && (lock->fl_type != F_UNLCK)
+                            && !same_owner (l, lock)) {
+                                ret = 0;
+                                break;
+                        }
+                }
+        }
+        return ret;
 }
 
 
@@ -331,231 +794,428 @@ extern void do_blocked_rw (pl_inode_t *);
 
 
 static void
-__insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock,
-		    gf_lk_domain_t dom)
+__insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-	posix_lock_t  *conf = NULL;
-	posix_lock_t  *t = NULL;
-	posix_lock_t  *sum = NULL;
-	int            i = 0;
-	struct _values v = { .locks = {0, 0, 0} };
-
-	list_for_each_entry_safe (conf, t, DOMAIN_HEAD (pl_inode, dom), list) {
-		if (!locks_overlap (conf, lock))
-			continue;
-
-		if (same_owner (conf, lock)) {
-			if (conf->fl_type == lock->fl_type) {
-				sum = add_locks (lock, conf);
-
-				sum->fl_type    = lock->fl_type;
-				sum->transport  = lock->transport;
-				sum->client_pid = lock->client_pid;
+        posix_lock_t  *conf = NULL;
+        posix_lock_t  *t = NULL;
+        posix_lock_t  *sum = NULL;
+        int            i = 0;
+        struct _values v = { .locks = {0, 0, 0} };
+
+        list_for_each_entry_safe (conf, t, &pl_inode->ext_list, list) {
+                if (conf->blocked)
+                        continue;
+                if (!locks_overlap (conf, lock))
+                        continue;
+
+                if (same_owner (conf, lock)) {
+                        if (conf->fl_type == lock->fl_type) {
+                                sum = add_locks (lock, conf);
+
+                                sum->fl_type    = lock->fl_type;
+                                sum->client     = lock->client;
+                                sum->fd_num     = lock->fd_num;
+                                sum->client_pid = lock->client_pid;
+                                sum->owner      = lock->owner;
+
+                                __delete_lock (pl_inode, conf);
+                                __destroy_lock (conf);
+
+                                __destroy_lock (lock);
+                                INIT_LIST_HEAD (&sum->list);
+                                posix_lock_to_flock (sum, &sum->user_flock);
+                                __insert_and_merge (pl_inode, sum);
+
+                                return;
+                        } else {
+                                sum = add_locks (lock, conf);
+
+                                sum->fl_type    = conf->fl_type;
+                                sum->client     = conf->client;
+                                sum->fd_num     = conf->fd_num;
+                                sum->client_pid = conf->client_pid;
+                                sum->owner      = conf->owner;
+
+                                v = subtract_locks (sum, lock);
+
+                                __delete_lock (pl_inode, conf);
+                                __destroy_lock (conf);
+
+                                __delete_lock (pl_inode, lock);
+                                __destroy_lock (lock);
+
+                                __destroy_lock (sum);
+
+                                for (i = 0; i < 3; i++) {
+                                        if (!v.locks[i])
+                                                continue;
+
+                                        INIT_LIST_HEAD (&v.locks[i]->list);
+                                        posix_lock_to_flock (v.locks[i],
+                                                       &v.locks[i]->user_flock);
+                                        __insert_and_merge (pl_inode,
+                                                            v.locks[i]);
+                                }
+
+                                __delete_unlck_locks (pl_inode);
+                                return;
+                        }
+                }
+
+                if (lock->fl_type == F_UNLCK) {
+                        continue;
+                }
+
+                if ((conf->fl_type == F_RDLCK) && (lock->fl_type == F_RDLCK)) {
+                        __insert_lock (pl_inode, lock);
+                        return;
+                }
+        }
+
+        /* no conflicts, so just insert */
+        if (lock->fl_type != F_UNLCK) {
+                __insert_lock (pl_inode, lock);
+        } else {
+                __destroy_lock (lock);
+        }
+}
 
-				__delete_lock (pl_inode, conf); 
-				__destroy_lock (conf);
 
-				__destroy_lock (lock);
-				__insert_and_merge (pl_inode, sum, dom);
+void
+__grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode, struct list_head *granted)
+{
+        struct list_head  tmp_list;
+        posix_lock_t     *l = NULL;
+        posix_lock_t     *tmp = NULL;
+        posix_lock_t     *conf = NULL;
+
+        INIT_LIST_HEAD (&tmp_list);
+
+        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
+                if (l->blocked) {
+                        conf = first_overlap (pl_inode, l);
+                        if (conf)
+                                continue;
+
+                        l->blocked = 0;
+                        list_move_tail (&l->list, &tmp_list);
+                }
+        }
+
+        list_for_each_entry_safe (l, tmp, &tmp_list, list) {
+                list_del_init (&l->list);
+
+                if (__is_lock_grantable (pl_inode, l)) {
+                        conf = GF_CALLOC (1, sizeof (*conf),
+                                          gf_locks_mt_posix_lock_t);
+
+                        if (!conf) {
+                                l->blocked = 1;
+                                __insert_lock (pl_inode, l);
+                                continue;
+                        }
+
+                        conf->frame = l->frame;
+                        l->frame = NULL;
+
+                        posix_lock_to_flock (l, &conf->user_flock);
+
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Granted",
+                                l->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                l->client_pid, lkowner_utoa (&l->owner),
+                                l->user_flock.l_start,
+                                l->user_flock.l_len);
+
+                        __insert_and_merge (pl_inode, l);
+
+                        list_add (&conf->list, granted);
+                } else {
+                        l->blocked = 1;
+                        __insert_lock (pl_inode, l);
+                }
+        }
+}
 
-				return;
-			} else {
-				sum = add_locks (lock, conf);
 
-				sum->fl_type    = conf->fl_type;
-				sum->transport  = conf->transport;
-				sum->client_pid = conf->client_pid;
+void
+grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode)
+{
+        struct list_head granted_list;
+        posix_lock_t     *tmp = NULL;
+        posix_lock_t     *lock = NULL;
 
-				v = subtract_locks (sum, lock);
-	
-				__delete_lock (pl_inode, conf);
-				__destroy_lock (conf);
+        INIT_LIST_HEAD (&granted_list);
 
-				__delete_lock (pl_inode, lock);
-				__destroy_lock (lock);
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __grant_blocked_locks (this, pl_inode, &granted_list);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
 
-				__destroy_lock (sum);
+        list_for_each_entry_safe (lock, tmp, &granted_list, list) {
+                list_del_init (&lock->list);
 
-				for (i = 0; i < 3; i++) {
-					if (!v.locks[i])
-						continue;
+                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
+                              &lock->user_flock, 0, 0, NULL);
 
-					if (v.locks[i]->fl_type == F_UNLCK) {
-						__destroy_lock (v.locks[i]);
-						continue;
-					}
-					__insert_and_merge (pl_inode,
-							    v.locks[i], dom);
-				}
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
+                                     &lock->user_flock, NULL);
 
-				__delete_unlck_locks (pl_inode, dom);
-				return; 
-			}
-		}
+                GF_FREE (lock);
+        }
 
-		if (lock->fl_type == F_UNLCK) {
-			continue;
-		}
+        return;
+}
 
-		if ((conf->fl_type == F_RDLCK) && (lock->fl_type == F_RDLCK)) {
-			pl_insert_lock (pl_inode, lock, dom);
-			return;
-		}
-	}
+static int
+pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
+                        posix_lock_t *old_lock)
+{
+        struct gf_flock  flock       = {0,};
+        posix_lock_t *unlock_lock = NULL;
 
-	/* no conflicts, so just insert */
-	if (lock->fl_type != F_UNLCK) {
-		pl_insert_lock (pl_inode, lock, dom);
-	} else {
-		__destroy_lock (lock);
-	}
-}
+        struct list_head granted_list;
+        posix_lock_t     *tmp = NULL;
+        posix_lock_t     *lock = NULL;
 
+        int ret = -1;
 
-void
-__grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode,
-		       gf_lk_domain_t dom, struct list_head *granted)
-{
-	struct list_head  tmp_list;
-	posix_lock_t     *l = NULL;
-	posix_lock_t     *tmp = NULL;
-	posix_lock_t     *conf = NULL;
+        INIT_LIST_HEAD (&granted_list);
 
-	INIT_LIST_HEAD (&tmp_list);
+        flock.l_type   = F_UNLCK;
+        flock.l_whence = old_lock->user_flock.l_whence;
+        flock.l_start  = old_lock->user_flock.l_start;
+        flock.l_len    = old_lock->user_flock.l_len;
 
-	list_for_each_entry_safe (l, tmp, DOMAIN_HEAD (pl_inode, dom), list) {
-		if (l->blocked) {
-			conf = first_overlap (pl_inode, l, dom);
-			if (conf)
-				continue;
 
-			l->blocked = 0;
-			list_move_tail (&l->list, &tmp_list);
-		}
-	}
+        unlock_lock = new_posix_lock (&flock, old_lock->client,
+                                      old_lock->client_pid, &old_lock->owner,
+                                      old_lock->fd);
+        GF_VALIDATE_OR_GOTO (this->name, unlock_lock, out);
+        ret = 0;
 
-	list_for_each_entry_safe (l, tmp, &tmp_list, list) {
-		list_del_init (&l->list);
+        __insert_and_merge (pl_inode, unlock_lock);
 
-		if (pl_is_lock_grantable (pl_inode, l, dom)) {
-			conf = CALLOC (1, sizeof (*conf));
+        __grant_blocked_locks (this, pl_inode, &granted_list);
 
-			if (!conf) {
-				l->blocked = 1;
-				pl_insert_lock (pl_inode, l, dom);
-				continue;
-			}
+        list_for_each_entry_safe (lock, tmp, &granted_list, list) {
+                list_del_init (&lock->list);
 
-			conf->frame = l->frame;
-			l->frame = NULL;
+                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
+                              &lock->user_flock, 0, 0, NULL);
 
-			posix_lock_to_flock (l, &conf->user_flock);
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
+                                     &lock->user_flock, NULL);
 
-			gf_log (this->name, GF_LOG_DEBUG,
-				"%s (pid=%d) %"PRId64" - %"PRId64" => Granted",
-				l->fl_type == F_UNLCK ? "Unlock" : "Lock",
-				l->client_pid,
-				l->user_flock.l_start,
-				l->user_flock.l_len);
+                GF_FREE (lock);
+        }
 
-			__insert_and_merge (pl_inode, l, dom);
+out:
+        return ret;
+}
 
-			list_add (&conf->list, granted);
-		} else {
-			l->blocked = 1;
-			pl_insert_lock (pl_inode, l, dom);
-		}
-	}
+int
+pl_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+          int can_block)
+{
+        int              ret = 0;
+
+        errno = 0;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                /* Send unlock before the actual lock to
+                   prevent lock upgrade / downgrade
+                   problems only if:
+                   - it is a blocking call
+                   - it has other conflicting locks
+                */
+
+                if (can_block &&
+                    !(__is_lock_grantable (pl_inode, lock))) {
+                        ret = pl_send_prelock_unlock (this, pl_inode,
+                                                      lock);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                "Could not send pre-lock "
+                                        "unlock");
+                }
+
+                if (__is_lock_grantable (pl_inode, lock)) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => OK",
+                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                lock->client_pid,
+                                lkowner_utoa (&lock->owner),
+                                lock->user_flock.l_start,
+                                lock->user_flock.l_len);
+                        __insert_and_merge (pl_inode, lock);
+                } else if (can_block) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
+                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                lock->client_pid,
+                                lkowner_utoa (&lock->owner),
+                                lock->user_flock.l_start,
+                                lock->user_flock.l_len);
+                        lock->blocked = 1;
+                        __insert_lock (pl_inode, lock);
+                        ret = -1;
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => NOK",
+                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                lock->client_pid,
+                                lkowner_utoa (&lock->owner),
+                                lock->user_flock.l_start,
+                                lock->user_flock.l_len);
+                        errno = EAGAIN;
+                        ret = -1;
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        grant_blocked_locks (this, pl_inode);
+
+        do_blocked_rw (pl_inode);
+
+        return ret;
 }
 
 
-void
-grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode, gf_lk_domain_t dom)
+posix_lock_t *
+pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-	struct list_head granted_list;
-	posix_lock_t     *tmp = NULL;
-	posix_lock_t     *lock = NULL;
-
-	INIT_LIST_HEAD (&granted_list);
+        posix_lock_t *conf = NULL;
 
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
-		__grant_blocked_locks (this, pl_inode, dom, &granted_list);
-	}
-	pthread_mutex_unlock (&pl_inode->mutex);
+        conf = first_conflicting_overlap (pl_inode, lock);
 
-	list_for_each_entry_safe (lock, tmp, &granted_list, list) {
-		list_del_init (&lock->list);
+        if (conf == NULL) {
+                lock->fl_type = F_UNLCK;
+                return lock;
+        }
 
-		STACK_UNWIND (lock->frame, 0, 0, &lock->user_flock);
+        return conf;
+}
 
-		FREE (lock);
-	}
 
-	return;
+struct _lock_table *
+pl_lock_table_new (void)
+{
+        struct _lock_table *new = NULL;
+
+        new = GF_CALLOC (1, sizeof (struct _lock_table), gf_common_mt_lock_table);
+        if (new == NULL) {
+                goto out;
+        }
+        INIT_LIST_HEAD (&new->entrylk_lockers);
+        INIT_LIST_HEAD (&new->inodelk_lockers);
+        LOCK_INIT (&new->lock);
+out:
+        return new;
 }
 
 
 int
-pl_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
-	  int can_block,  gf_lk_domain_t dom)
-{
-	int              ret = 0;
-
-	errno = 0;
-
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
-		if (pl_is_lock_grantable (pl_inode, lock, dom)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"%s (pid=%d) %"PRId64" - %"PRId64" => OK",
-				lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-				lock->client_pid,
-				lock->user_flock.l_start,
-				lock->user_flock.l_len);
-			__insert_and_merge (pl_inode, lock, dom);
-		} else if (can_block) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"%s (pid=%d) %"PRId64" - %"PRId64" => Blocked",
-				lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-				lock->client_pid,
-				lock->user_flock.l_start,
-				lock->user_flock.l_len);
-			lock->blocked = 1;
-			pl_insert_lock (pl_inode, lock, dom);
-			ret = -1;
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"%s (pid=%d) %"PRId64" - %"PRId64" => NOK",
-				lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-				lock->client_pid,
-				lock->user_flock.l_start,
-				lock->user_flock.l_len);
-			errno = EAGAIN;
-			ret = -1;
-		}
-	}
-	pthread_mutex_unlock (&pl_inode->mutex);
-
-	grant_blocked_locks (this, pl_inode, dom);
-
-	do_blocked_rw (pl_inode);
-
-	return ret;
+pl_add_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc, fd_t *fd, pid_t pid, gf_lkowner_t *owner,
+               glusterfs_fop_t type)
+{
+        int32_t         ret = -1;
+        struct _locker *new = NULL;
+
+        GF_VALIDATE_OR_GOTO ("lock-table", table, out);
+        GF_VALIDATE_OR_GOTO ("lock-table", volume, out);
+
+        new = GF_CALLOC (1, sizeof (struct _locker), gf_common_mt_locker);
+        if (new == NULL) {
+                goto out;
+        }
+        INIT_LIST_HEAD (&new->lockers);
+
+        new->volume = gf_strdup (volume);
+
+        if (fd == NULL) {
+                loc_copy (&new->loc, loc);
+        } else {
+                new->fd = fd_ref (fd);
+        }
+
+        new->pid   = pid;
+        new->owner = *owner;
+
+        LOCK (&table->lock);
+        {
+                if (type == GF_FOP_ENTRYLK)
+                        list_add_tail (&new->lockers, &table->entrylk_lockers);
+                else
+                        list_add_tail (&new->lockers, &table->inodelk_lockers);
+        }
+        UNLOCK (&table->lock);
+out:
+        return ret;
 }
 
-
-posix_lock_t *
-pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock, gf_lk_domain_t dom)
+int
+pl_del_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc, fd_t *fd, gf_lkowner_t *owner, glusterfs_fop_t type)
 {
-	posix_lock_t *conf = NULL;
-
-	conf = first_overlap (pl_inode, lock, dom);
-
-	if (conf == NULL) {
-		lock->fl_type = F_UNLCK;
-		return lock;
-	}
+        struct _locker    *locker = NULL;
+        struct _locker    *tmp = NULL;
+        int32_t            ret = -1;
+        struct list_head  *head = NULL;
+        struct list_head   del;
+
+        GF_VALIDATE_OR_GOTO ("lock-table", table, out);
+        GF_VALIDATE_OR_GOTO ("lock-table", volume, out);
+
+        INIT_LIST_HEAD (&del);
+
+        LOCK (&table->lock);
+        {
+                if (type == GF_FOP_ENTRYLK) {
+                        head = &table->entrylk_lockers;
+                } else {
+                        head = &table->inodelk_lockers;
+                }
+
+                list_for_each_entry_safe (locker, tmp, head, lockers) {
+                        if (!is_same_lkowner (&locker->owner, owner) ||
+                            strcmp (locker->volume, volume))
+                                continue;
+
+                        /*
+                         * It is possible for inodelk lock to come on anon-fd
+                         * and inodelk unlock to come on normal fd in case of
+                         * client re-opens. So don't check for fds to be equal.
+                         */
+                        if (locker->fd && fd)
+                                list_move_tail (&locker->lockers, &del);
+                        else if (locker->loc.inode && loc &&
+                                 (locker->loc.inode == loc->inode))
+                                list_move_tail (&locker->lockers, &del);
+                }
+        }
+        UNLOCK (&table->lock);
+
+        tmp = NULL;
+        locker = NULL;
+
+        list_for_each_entry_safe (locker, tmp, &del, lockers) {
+                list_del_init (&locker->lockers);
+                if (locker->fd)
+                        fd_unref (locker->fd);
+                else
+                        loc_wipe (&locker->loc);
+
+                GF_FREE (locker->volume);
+                GF_FREE (locker);
+        }
+
+        ret = 0;
+out:
+        return ret;
 
-	return conf;
 }
+
diff --git a/xlators/features/locks/src/common.h b/xlators/features/locks/src/common.h
index 135f33011..db19ec978 100644
--- a/xlators/features/locks/src/common.h
+++ b/xlators/features/locks/src/common.h
@@ -1,50 +1,71 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __COMMON_H__
 #define __COMMON_H__
 
+#include "lkowner.h"
+/*dump locks format strings */
+#define RANGE_FMT               "type=%s, whence=%hd, start=%llu, len=%llu"
+#define ENTRY_FMT               "type=%s on basename=%s"
+#define DUMP_GEN_FMT            "pid = %llu, owner=%s, client=%p"
+#define GRNTD_AT                "granted at %s"
+#define BLKD_AT                 "blocked at %s"
+#define CONN_ID                 "connection-id=%s"
+#define DUMP_BLKD_FMT           DUMP_GEN_FMT", "CONN_ID", "BLKD_AT
+#define DUMP_GRNTD_FMT          DUMP_GEN_FMT", "CONN_ID", "GRNTD_AT
+#define DUMP_BLKD_GRNTD_FMT     DUMP_GEN_FMT", "CONN_ID", "BLKD_AT", "GRNTD_AT
+
+#define ENTRY_BLKD_FMT          ENTRY_FMT", "DUMP_BLKD_FMT
+#define ENTRY_GRNTD_FMT         ENTRY_FMT", "DUMP_GRNTD_FMT
+#define ENTRY_BLKD_GRNTD_FMT    ENTRY_FMT", "DUMP_BLKD_GRNTD_FMT
+
+#define RANGE_BLKD_FMT          RANGE_FMT", "DUMP_BLKD_FMT
+#define RANGE_GRNTD_FMT         RANGE_FMT", "DUMP_GRNTD_FMT
+#define RANGE_BLKD_GRNTD_FMT    RANGE_FMT", "DUMP_BLKD_GRNTD_FMT
+
+#define SET_FLOCK_PID(flock, lock) ((flock)->l_pid = lock->client_pid)
+
+struct _locker {
+        struct list_head  lockers;
+        char             *volume;
+        loc_t             loc;
+        fd_t             *fd;
+        gf_lkowner_t      owner;
+        pid_t             pid;
+};
+
+struct _lock_table {
+        struct list_head  inodelk_lockers;
+        struct list_head  entrylk_lockers;
+        gf_lock_t         lock;
+};
+
 posix_lock_t *
-new_posix_lock (struct flock *flock, transport_t *transport, pid_t client_pid);
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
+                gf_lkowner_t *owner, fd_t *fd);
 
 pl_inode_t *
 pl_inode_get (xlator_t *this, inode_t *inode);
 
 posix_lock_t *
-pl_getlk (pl_inode_t *inode, posix_lock_t *lock, gf_lk_domain_t domain);
+pl_getlk (pl_inode_t *inode, posix_lock_t *lock);
 
 int
 pl_setlk (xlator_t *this, pl_inode_t *inode, posix_lock_t *lock,
-	  int can_block, gf_lk_domain_t domain);
-
-int
-pl_is_lock_grantable (pl_inode_t *pl_inode, posix_lock_t *lock,
-		      gf_lk_domain_t dom);
+          int can_block);
 
 void
-pl_insert_lock (pl_inode_t *pl_inode, posix_lock_t *lock, gf_lk_domain_t dom);
+grant_blocked_locks (xlator_t *this, pl_inode_t *inode);
 
 void
-grant_blocked_locks (xlator_t *this, pl_inode_t *inode, gf_lk_domain_t domain);
-
-void
-posix_lock_to_flock (posix_lock_t *lock, struct flock *flock);
+posix_lock_to_flock (posix_lock_t *lock, struct gf_flock *flock);
 
 int
 locks_overlap (posix_lock_t *l1, posix_lock_t *l2);
@@ -56,4 +77,111 @@ void __delete_lock (pl_inode_t *, posix_lock_t *);
 
 void __destroy_lock (posix_lock_t *);
 
+pl_dom_list_t *
+get_domain (pl_inode_t *pl_inode, const char *volume);
+
+void
+grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_dom_list_t *dom);
+
+void
+__delete_inode_lock (pl_inode_lock_t *lock);
+
+void
+__pl_inodelk_unref (pl_inode_lock_t *lock);
+
+void
+grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_entry_lock_t *unlocked, pl_dom_list_t *dom);
+
+void pl_update_refkeeper (xlator_t *this, inode_t *inode);
+
+int32_t
+__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname);
+int32_t
+get_inodelk_count (xlator_t *this, inode_t *inode, char *domname);
+
+int32_t
+__get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode);
+int32_t
+get_entrylk_count (xlator_t *this, inode_t *inode);
+
+void pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+                  int cmd, struct gf_flock *flock, const char *domain);
+
+void pl_trace_out (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+                   int cmd, struct gf_flock *flock, int op_ret, int op_errno, const char *domain);
+
+void pl_trace_block (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+                     int cmd, struct gf_flock *flock, const char *domain);
+
+void pl_trace_flush (xlator_t *this, call_frame_t *frame, fd_t *fd);
+
+void entrylk_trace_in (xlator_t *this, call_frame_t *frame, const char *volume,
+                       fd_t *fd, loc_t *loc, const char *basename,
+                       entrylk_cmd cmd, entrylk_type type);
+
+void entrylk_trace_out (xlator_t *this, call_frame_t *frame, const char *volume,
+                        fd_t *fd, loc_t *loc, const char *basename,
+                        entrylk_cmd cmd, entrylk_type type,
+                        int op_ret, int op_errno);
+
+void entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
+                          fd_t *fd, loc_t *loc, const char *basename,
+                          entrylk_cmd cmd, entrylk_type type);
+
+void
+pl_print_verdict (char *str, int size, int op_ret, int op_errno);
+
+void
+pl_print_lockee (char *str, int size, fd_t *fd, loc_t *loc);
+
+void
+pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame);
+
+void
+pl_print_inodelk (char *str, int size, int cmd, struct gf_flock *flock, const char *domain);
+
+void
+pl_trace_release (xlator_t *this, fd_t *fd);
+
+unsigned long
+fd_to_fdnum (fd_t *fd);
+
+fd_t *
+fd_from_fdnum (posix_lock_t *lock);
+
+int
+pl_reserve_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                  int can_block);
+int
+reservelks_equal (posix_lock_t *l1, posix_lock_t *l2);
+
+int
+pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
+                     posix_lock_t *lock, int can_block);
+int
+pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *reqlock);
+
+uint32_t
+check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename);
+
+int32_t
+pl_add_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc,
+               fd_t *fd,
+               pid_t pid,
+               gf_lkowner_t *owner,
+               glusterfs_fop_t type);
+
+int32_t
+pl_del_locker (struct _lock_table *table, const char *volume,
+               loc_t *loc,
+               fd_t *fd,
+               gf_lkowner_t *owner,
+               glusterfs_fop_t type);
+
+struct _lock_table *
+pl_lock_table_new (void);
+
 #endif /* __COMMON_H__ */
diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
new file mode 100644
index 000000000..0785dc547
--- /dev/null
+++ b/xlators/features/locks/src/entrylk.c
@@ -0,0 +1,848 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "compat.h"
+#include "xlator.h"
+#include "inode.h"
+#include "logging.h"
+#include "common-utils.h"
+#include "list.h"
+
+#include "locks.h"
+#include "common.h"
+
+static pl_entry_lock_t *
+new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
+                  client_t *client, pid_t client_pid, gf_lkowner_t *owner,
+                  const char *volume)
+
+{
+        pl_entry_lock_t *newlock = NULL;
+
+        newlock = GF_CALLOC (1, sizeof (pl_entry_lock_t),
+                             gf_locks_mt_pl_entry_lock_t);
+        if (!newlock) {
+                goto out;
+        }
+
+        newlock->basename   = basename ? gf_strdup (basename) : NULL;
+        newlock->type       = type;
+        newlock->trans      = client;
+        newlock->volume     = volume;
+        newlock->client_pid = client_pid;
+        newlock->owner      = *owner;
+
+        INIT_LIST_HEAD (&newlock->domain_list);
+        INIT_LIST_HEAD (&newlock->blocked_locks);
+
+out:
+        return newlock;
+}
+
+
+/**
+ * all_names - does a basename represent all names?
+ * @basename: name to check
+ */
+
+#define all_names(basename) ((basename == NULL) ? 1 : 0)
+
+/**
+ * names_conflict - do two names conflict?
+ * @n1: name
+ * @n2: name
+ */
+
+static int
+names_conflict (const char *n1, const char *n2)
+{
+        return all_names (n1) || all_names (n2) || !strcmp (n1, n2);
+}
+
+
+static inline int
+__same_entrylk_owner (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
+{
+
+        return (is_same_lkowner (&l1->owner, &l2->owner) &&
+                (l1->trans  == l2->trans));
+}
+
+
+/**
+ * lock_grantable - is this lock grantable?
+ * @inode: inode in which to look
+ * @basename: name we're trying to lock
+ * @type: type of lock
+ */
+static pl_entry_lock_t *
+__lock_grantable (pl_dom_list_t *dom, const char *basename, entrylk_type type)
+{
+        pl_entry_lock_t *lock = NULL;
+
+        if (list_empty (&dom->entrylk_list))
+                return NULL;
+
+        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
+                if (names_conflict (lock->basename, basename))
+                        return lock;
+        }
+
+        return NULL;
+}
+
+static pl_entry_lock_t *
+__blocked_lock_conflict (pl_dom_list_t *dom, const char *basename, entrylk_type type)
+{
+        pl_entry_lock_t *lock = NULL;
+
+        if (list_empty (&dom->blocked_entrylks))
+                return NULL;
+
+        list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
+                if (names_conflict (lock->basename, basename))
+                        return lock;
+        }
+
+        return NULL;
+}
+
+static int
+__owner_has_lock (pl_dom_list_t *dom, pl_entry_lock_t *newlock)
+{
+        pl_entry_lock_t *lock = NULL;
+
+        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
+                if (__same_entrylk_owner (lock, newlock))
+                        return 1;
+        }
+
+        list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
+                if (__same_entrylk_owner (lock, newlock))
+                        return 1;
+        }
+
+        return 0;
+}
+
+static int
+names_equal (const char *n1, const char *n2)
+{
+        return (n1 == NULL && n2 == NULL) || (n1 && n2 && !strcmp (n1, n2));
+}
+
+void
+pl_print_entrylk (char *str, int size, entrylk_cmd cmd, entrylk_type type,
+                  const char *basename, const char *domain)
+{
+        char *cmd_str = NULL;
+        char *type_str = NULL;
+
+        switch (cmd) {
+        case ENTRYLK_LOCK:
+                cmd_str = "LOCK";
+                break;
+
+        case ENTRYLK_LOCK_NB:
+                cmd_str = "LOCK_NB";
+                break;
+
+        case ENTRYLK_UNLOCK:
+                cmd_str = "UNLOCK";
+                break;
+
+        default:
+                cmd_str = "UNKNOWN";
+                break;
+        }
+
+        switch (type) {
+        case ENTRYLK_RDLCK:
+                type_str = "READ";
+                break;
+        case ENTRYLK_WRLCK:
+                type_str = "WRITE";
+                break;
+        default:
+                type_str = "UNKNOWN";
+                break;
+        }
+
+        snprintf (str, size, "lock=ENTRYLK, cmd=%s, type=%s, basename=%s, domain: %s",
+                  cmd_str, type_str, basename, domain);
+}
+
+
+void
+entrylk_trace_in (xlator_t *this, call_frame_t *frame, const char *domain,
+                  fd_t *fd, loc_t *loc, const char *basename,
+                  entrylk_cmd cmd, entrylk_type type)
+{
+        posix_locks_private_t  *priv = NULL;
+        char                    pl_locker[256];
+        char                    pl_lockee[256];
+        char                    pl_entrylk[256];
+
+        priv = this->private;
+
+        if (!priv->trace)
+                return;
+
+        pl_print_locker (pl_locker, 256, this, frame);
+        pl_print_lockee (pl_lockee, 256, fd, loc);
+        pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, domain);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}",
+                pl_locker, pl_lockee, pl_entrylk);
+}
+
+
+void
+entrylk_trace_out (xlator_t *this, call_frame_t *frame, const char *domain,
+                   fd_t *fd, loc_t *loc, const char *basename,
+                   entrylk_cmd cmd, entrylk_type type, int op_ret, int op_errno)
+{
+        posix_locks_private_t  *priv = NULL;
+        char                    pl_locker[256];
+        char                    pl_lockee[256];
+        char                    pl_entrylk[256];
+        char                    verdict[32];
+
+        priv = this->private;
+
+        if (!priv->trace)
+                return;
+
+        pl_print_locker (pl_locker, 256, this, frame);
+        pl_print_lockee (pl_lockee, 256, fd, loc);
+        pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, domain);
+        pl_print_verdict (verdict, 32, op_ret, op_errno);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}",
+                verdict, pl_locker, pl_lockee, pl_entrylk);
+}
+
+
+void
+entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
+                     fd_t *fd, loc_t *loc, const char *basename,
+                     entrylk_cmd cmd, entrylk_type type)
+
+{
+        posix_locks_private_t  *priv = NULL;
+        char                    pl_locker[256];
+        char                    pl_lockee[256];
+        char                    pl_entrylk[256];
+
+        priv = this->private;
+
+        if (!priv->trace)
+                return;
+
+        pl_print_locker (pl_locker, 256, this, frame);
+        pl_print_lockee (pl_lockee, 256, fd, loc);
+        pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, volume);
+
+        gf_log (this->name, GF_LOG_INFO,
+                "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}",
+                pl_locker, pl_lockee, pl_entrylk);
+}
+
+/**
+ * __find_most_matching_lock - find the lock struct which most matches in order of:
+ *                           lock on the exact basename ||
+ *                           an all_names lock
+ *
+ *
+ * @inode: inode in which to look
+ * @basename: name to search for
+ */
+
+static pl_entry_lock_t *
+__find_most_matching_lock (pl_dom_list_t *dom, const char *basename)
+{
+        pl_entry_lock_t *lock;
+        pl_entry_lock_t *all = NULL;
+        pl_entry_lock_t *exact = NULL;
+
+        if (list_empty (&dom->entrylk_list))
+                return NULL;
+
+        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
+                if (all_names (lock->basename))
+                        all = lock;
+                else if (names_equal (lock->basename, basename))
+                        exact = lock;
+        }
+
+        return (exact ? exact : all);
+}
+
+/**
+ * __lock_name - lock a name in a directory
+ * @inode: inode for the directory in which to lock
+ * @basename: name of the entry to lock
+ *            if null, lock the entire directory
+ *
+ * the entire directory being locked is represented as: a single
+ * pl_entry_lock_t present in the entrylk_locks list with its
+ * basename = NULL
+ */
+
+int
+__lock_name (pl_inode_t *pinode, const char *basename, entrylk_type type,
+             call_frame_t *frame, pl_dom_list_t *dom, xlator_t *this,
+             int nonblock, char *conn_id)
+{
+        pl_entry_lock_t *lock       = NULL;
+        pl_entry_lock_t *conf       = NULL;
+        int              ret        = -EINVAL;
+
+        lock = new_entrylk_lock (pinode, basename, type,
+                                 frame->root->client, frame->root->pid,
+                                 &frame->root->lk_owner, dom->domain);
+        if (!lock) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        lock->frame   = frame;
+        lock->this    = this;
+        lock->trans   = frame->root->client;
+
+        if (conn_id) {
+                lock->connection_id = gf_strdup (conn_id);
+        }
+
+        conf = __lock_grantable (dom, basename, type);
+        if (conf) {
+                ret = -EAGAIN;
+                if (nonblock){
+                        GF_FREE (lock->connection_id);
+                        GF_FREE ((char *)lock->basename);
+                        GF_FREE (lock);
+                        goto out;
+
+                }
+
+                gettimeofday (&lock->blkd_time, NULL);
+                list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Blocking lock: {pinode=%p, basename=%s}",
+                        pinode, basename);
+
+                goto out;
+        }
+
+        if ( __blocked_lock_conflict (dom, basename, type) && !(__owner_has_lock (dom, lock))) {
+                ret = -EAGAIN;
+                if (nonblock) {
+                        GF_FREE (lock->connection_id);
+                        GF_FREE ((char *) lock->basename);
+                        GF_FREE (lock);
+                        goto out;
+
+                }
+                lock->frame     = frame;
+                lock->this      = this;
+
+                gettimeofday (&lock->blkd_time, NULL);
+                list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Lock is grantable, but blocking to prevent starvation");
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Blocking lock: {pinode=%p, basename=%s}",
+                        pinode, basename);
+
+                ret = -EAGAIN;
+                goto out;
+        }
+        switch (type) {
+
+        case ENTRYLK_WRLCK:
+                gettimeofday (&lock->granted_time, NULL);
+                list_add_tail (&lock->domain_list, &dom->entrylk_list);
+                break;
+
+        default:
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Invalid type for entrylk specified: %d", type);
+                ret = -EINVAL;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/**
+ * __unlock_name - unlock a name in a directory
+ * @inode: inode for the directory to unlock in
+ * @basename: name of the entry to unlock
+ *            if null, unlock the entire directory
+ */
+
+pl_entry_lock_t *
+__unlock_name (pl_dom_list_t *dom, const char *basename, entrylk_type type)
+{
+        pl_entry_lock_t *lock = NULL;
+        pl_entry_lock_t *ret_lock = NULL;
+
+        lock = __find_most_matching_lock (dom, basename);
+
+        if (!lock) {
+                gf_log ("locks", GF_LOG_DEBUG,
+                        "unlock on %s (type=ENTRYLK_WRLCK) attempted but no matching lock found",
+                        basename);
+                goto out;
+        }
+
+        if (names_equal (lock->basename, basename)
+            && lock->type == type) {
+
+                if (type == ENTRYLK_WRLCK) {
+                        list_del_init (&lock->domain_list);
+                        ret_lock = lock;
+                }
+        } else {
+                gf_log ("locks", GF_LOG_DEBUG,
+                        "Unlock for a non-existing lock!");
+                goto out;
+        }
+
+out:
+        return ret_lock;
+}
+
+uint32_t
+check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename)
+{
+        uint32_t        entrylk = 0;
+        pl_inode_t      *pinode = 0;
+        pl_dom_list_t   *dom = NULL;
+        pl_entry_lock_t *conf       = NULL;
+
+        pinode = pl_inode_get (this, parent);
+        if (!pinode)
+                goto out;
+        pthread_mutex_lock (&pinode->mutex);
+        {
+                list_for_each_entry (dom, &pinode->dom_list, inode_list) {
+                        conf = __lock_grantable (dom, basename, ENTRYLK_WRLCK);
+                        if (conf && conf->basename) {
+                                entrylk = 1;
+                                break;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pinode->mutex);
+
+out:
+        return entrylk;
+}
+
+void
+__grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
+                             pl_dom_list_t *dom, struct list_head *granted)
+{
+        int              bl_ret = 0;
+        pl_entry_lock_t *bl   = NULL;
+        pl_entry_lock_t *tmp  = NULL;
+
+        struct list_head blocked_list;
+
+        INIT_LIST_HEAD (&blocked_list);
+        list_splice_init (&dom->blocked_entrylks, &blocked_list);
+
+        list_for_each_entry_safe (bl, tmp, &blocked_list,
+                                  blocked_locks) {
+
+                list_del_init (&bl->blocked_locks);
+
+
+                gf_log ("locks", GF_LOG_TRACE,
+                        "Trying to unblock: {pinode=%p, basename=%s}",
+                        pl_inode, bl->basename);
+
+                bl_ret = __lock_name (pl_inode, bl->basename, bl->type,
+                                      bl->frame, dom, bl->this, 0,
+                                      bl->connection_id);
+
+                if (bl_ret == 0) {
+                        list_add (&bl->blocked_locks, granted);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "should never happen");
+                        GF_FREE (bl->connection_id);
+                        GF_FREE ((char *)bl->basename);
+                        GF_FREE (bl);
+                }
+        }
+        return;
+}
+
+/* Grants locks if possible which are blocked on a lock */
+void
+grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_entry_lock_t *unlocked, pl_dom_list_t *dom)
+{
+        struct list_head  granted_list;
+        pl_entry_lock_t  *tmp = NULL;
+        pl_entry_lock_t  *lock = NULL;
+
+        INIT_LIST_HEAD (&granted_list);
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __grant_blocked_entry_locks (this, pl_inode, dom,
+                                             &granted_list);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
+                list_del_init (&lock->blocked_locks);
+
+                entrylk_trace_out (this, lock->frame, NULL, NULL, NULL,
+                                   lock->basename, ENTRYLK_LOCK, lock->type,
+                                   0, 0);
+
+                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0, NULL);
+
+                GF_FREE (lock->connection_id);
+                GF_FREE ((char *)lock->basename);
+                GF_FREE (lock);
+        }
+
+        GF_FREE ((char *)unlocked->basename);
+        GF_FREE (unlocked->connection_id);
+        GF_FREE (unlocked);
+
+        return;
+}
+
+/**
+ * release_entry_locks_for_client: release all entry locks from this
+ * client for this loc_t
+ */
+
+static int
+release_entry_locks_for_client (xlator_t *this, pl_inode_t *pinode,
+                                pl_dom_list_t *dom, client_t *client)
+{
+        pl_entry_lock_t  *lock = NULL;
+        pl_entry_lock_t  *tmp = NULL;
+        struct list_head  granted;
+        struct list_head  released;
+
+        INIT_LIST_HEAD (&granted);
+        INIT_LIST_HEAD (&released);
+
+        pthread_mutex_lock (&pinode->mutex);
+        {
+                list_for_each_entry_safe (lock, tmp, &dom->blocked_entrylks,
+                                          blocked_locks) {
+                        if (lock->trans != client)
+                                continue;
+
+                        list_del_init (&lock->blocked_locks);
+
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "releasing lock on  held by "
+                                "{client=%p}", client);
+
+                        list_add (&lock->blocked_locks, &released);
+
+                }
+
+                list_for_each_entry_safe (lock, tmp, &dom->entrylk_list,
+                                          domain_list) {
+                        if (lock->trans != client)
+                                continue;
+
+                        list_del_init (&lock->domain_list);
+
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "releasing lock on  held by "
+                                "{client=%p}", client);
+
+                        GF_FREE ((char *)lock->basename);
+                        GF_FREE (lock->connection_id);
+                        GF_FREE (lock);
+                }
+
+                __grant_blocked_entry_locks (this, pinode, dom, &granted);
+
+        }
+
+        pthread_mutex_unlock (&pinode->mutex);
+
+        list_for_each_entry_safe (lock, tmp, &released, blocked_locks) {
+                list_del_init (&lock->blocked_locks);
+
+                STACK_UNWIND_STRICT (entrylk, lock->frame, -1, EAGAIN, NULL);
+
+                GF_FREE ((char *)lock->basename);
+                GF_FREE (lock->connection_id);
+                GF_FREE (lock);
+
+        }
+
+        list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
+                list_del_init (&lock->blocked_locks);
+
+                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0, NULL);
+
+                GF_FREE ((char *)lock->basename);
+                GF_FREE (lock->connection_id);
+                GF_FREE (lock);
+        }
+
+        return 0;
+}
+
+/* Common entrylk code called by pl_entrylk and pl_fentrylk */
+int
+pl_common_entrylk (call_frame_t *frame, xlator_t *this,
+                   const char *volume, inode_t *inode, const char *basename,
+                   entrylk_cmd cmd, entrylk_type type, loc_t *loc, fd_t *fd,
+                   dict_t *xdata)
+
+{
+        int32_t          op_ret   = -1;
+        int32_t          op_errno = 0;
+        int              ret      = -1;
+        char             unwind   = 1;
+        GF_UNUSED int    dict_ret = -1;
+        pl_inode_t      *pinode   = NULL;
+        pl_entry_lock_t *unlocked = NULL;
+        pl_dom_list_t   *dom      = NULL;
+        char            *conn_id  = NULL;
+        pl_ctx_t        *ctx      = NULL;
+
+        if (xdata)
+                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
+
+        pinode = pl_inode_get (this, inode);
+        if (!pinode) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        dom = get_domain (pinode, volume);
+        if (!dom){
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        entrylk_trace_in (this, frame, volume, fd, loc, basename, cmd, type);
+
+        if (frame->root->lk_owner.len == 0) {
+                /*
+                  this is a special case that means release
+                  all locks from this client
+                */
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Releasing locks for client %p", frame->root->client);
+
+                release_entry_locks_for_client (this, pinode, dom,
+                                                frame->root->client);
+                op_ret = 0;
+
+                goto out;
+        }
+
+        switch (cmd) {
+        case ENTRYLK_LOCK:
+                pthread_mutex_lock (&pinode->mutex);
+                {
+                        ret = __lock_name (pinode, basename, type,
+                                           frame, dom, this, 0, conn_id);
+                }
+                pthread_mutex_unlock (&pinode->mutex);
+
+                op_errno = -ret;
+                if (ret < 0) {
+                        if (ret == -EAGAIN)
+                                unwind = 0;
+                        else
+                                unwind = 1;
+                        goto out;
+                } else {
+                        op_ret = 0;
+                        op_errno = 0;
+                        unwind = 1;
+                        goto out;
+                }
+
+                break;
+
+        case ENTRYLK_LOCK_NB:
+                unwind = 1;
+                pthread_mutex_lock (&pinode->mutex);
+                {
+                        ret = __lock_name (pinode, basename, type,
+                                           frame, dom, this, 1, conn_id);
+                }
+                pthread_mutex_unlock (&pinode->mutex);
+
+                if (ret < 0) {
+                        op_errno = -ret;
+                        goto out;
+                }
+
+                break;
+
+        case ENTRYLK_UNLOCK:
+                pthread_mutex_lock (&pinode->mutex);
+                {
+                        unlocked = __unlock_name (dom, basename, type);
+                }
+                pthread_mutex_unlock (&pinode->mutex);
+
+                if (unlocked)
+                        grant_blocked_entry_locks (this, pinode, unlocked, dom);
+
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unexpected case in entrylk (cmd=%d). Please file"
+                        "a bug report at http://bugs.gluster.com", cmd);
+                goto out;
+        }
+
+        op_ret = 0;
+out:
+        pl_update_refkeeper (this, inode);
+        if (unwind) {
+                entrylk_trace_out (this, frame, volume, fd, loc, basename,
+                                   cmd, type, op_ret, op_errno);
+
+                ctx = pl_ctx_get (frame->root->client, this);
+
+                if (ctx == NULL) {
+                        gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+                        goto unwind;
+                }
+
+                if (cmd == ENTRYLK_UNLOCK)
+                        pl_del_locker (ctx->ltable, volume, loc, fd,
+                                       &frame->root->lk_owner,
+                                       GF_FOP_ENTRYLK);
+                else
+                        pl_add_locker (ctx->ltable, volume, loc, fd,
+                                       frame->root->pid,
+                                       &frame->root->lk_owner,
+                                       GF_FOP_ENTRYLK);
+
+unwind:
+                STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, NULL);
+        } else {
+                entrylk_trace_block (this, frame, volume, fd, loc, basename,
+                                     cmd, type);
+        }
+
+
+        return 0;
+}
+
+/**
+ * pl_entrylk:
+ *
+ * Locking on names (directory entries)
+ */
+
+int
+pl_entrylk (call_frame_t *frame, xlator_t *this,
+            const char *volume, loc_t *loc, const char *basename,
+            entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+{
+        pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd,
+                           type, loc, NULL, xdata);
+
+        return 0;
+}
+
+
+/**
+ * pl_fentrylk:
+ *
+ * Locking on names (directory entries)
+ */
+
+int
+pl_fentrylk (call_frame_t *frame, xlator_t *this,
+             const char *volume, fd_t *fd, const char *basename,
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+{
+        pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd,
+                           type, NULL, fd, xdata);
+
+        return 0;
+}
+
+
+int32_t
+__get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode)
+{
+        int32_t            count = 0;
+        pl_entry_lock_t   *lock  = NULL;
+        pl_dom_list_t     *dom   = NULL;
+
+        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+                list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
+                        count++;
+                }
+
+                list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
+                        count++;
+                }
+
+        }
+
+        return count;
+}
+
+int32_t
+get_entrylk_count (xlator_t *this, inode_t *inode)
+{
+        pl_inode_t   *pl_inode = NULL;
+        uint64_t      tmp_pl_inode = 0;
+        int           ret      = 0;
+        int32_t       count    = 0;
+
+        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
+        if (ret != 0) {
+                goto out;
+        }
+
+        pl_inode = (pl_inode_t *)(long) tmp_pl_inode;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                count = __get_entrylk_count (this, pl_inode);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+out:
+        return count;
+}
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
new file mode 100644
index 000000000..508523e11
--- /dev/null
+++ b/xlators/features/locks/src/inodelk.c
@@ -0,0 +1,825 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "compat.h"
+#include "xlator.h"
+#include "inode.h"
+#include "logging.h"
+#include "common-utils.h"
+#include "list.h"
+
+#include "locks.h"
+#include "common.h"
+
+inline void
+__delete_inode_lock (pl_inode_lock_t *lock)
+{
+        list_del (&lock->list);
+}
+
+static inline void
+__pl_inodelk_ref (pl_inode_lock_t *lock)
+{
+        lock->ref++;
+}
+
+inline void
+__pl_inodelk_unref (pl_inode_lock_t *lock)
+{
+        lock->ref--;
+        if (!lock->ref) {
+                GF_FREE (lock->connection_id);
+                GF_FREE (lock);
+        }
+}
+
+/* Check if 2 inodelks are conflicting on type. Only 2 shared locks don't conflict */
+static inline int
+inodelk_type_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+{
+        if (l2->fl_type == F_WRLCK || l1->fl_type == F_WRLCK)
+                return 1;
+
+        return 0;
+}
+
+void
+pl_print_inodelk (char *str, int size, int cmd, struct gf_flock *flock, const char *domain)
+{
+        char *cmd_str = NULL;
+        char *type_str = NULL;
+
+        switch (cmd) {
+#if F_GETLK != F_GETLK64
+        case F_GETLK64:
+#endif
+        case F_GETLK:
+                cmd_str = "GETLK";
+                break;
+
+#if F_SETLK != F_SETLK64
+        case F_SETLK64:
+#endif
+        case F_SETLK:
+                cmd_str = "SETLK";
+                break;
+
+#if F_SETLKW != F_SETLKW64
+        case F_SETLKW64:
+#endif
+        case F_SETLKW:
+                cmd_str = "SETLKW";
+                break;
+
+        default:
+                cmd_str = "UNKNOWN";
+                break;
+        }
+
+        switch (flock->l_type) {
+        case F_RDLCK:
+                type_str = "READ";
+                break;
+        case F_WRLCK:
+                type_str = "WRITE";
+                break;
+        case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+        default:
+                type_str = "UNKNOWN";
+                break;
+        }
+
+        snprintf (str, size, "lock=INODELK, cmd=%s, type=%s, "
+                  "domain: %s, start=%llu, len=%llu, pid=%llu",
+                  cmd_str, type_str, domain,
+                  (unsigned long long) flock->l_start,
+                  (unsigned long long) flock->l_len,
+                  (unsigned long long) flock->l_pid);
+}
+
+/* Determine if the two inodelks overlap reach other's lock regions */
+static int
+inodelk_overlap (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+{
+        return ((l1->fl_end >= l2->fl_start) &&
+                (l2->fl_end >= l1->fl_start));
+}
+
+/* Returns true if the 2 inodelks have the same owner */
+static inline int
+same_inodelk_owner (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+{
+        return (is_same_lkowner (&l1->owner, &l2->owner) &&
+                (l1->client == l2->client));
+}
+
+/* Returns true if the 2 inodelks conflict with each other */
+static int
+inodelk_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+{
+        return (inodelk_overlap (l1, l2) &&
+                inodelk_type_conflict (l1, l2));
+}
+
+/* Determine if lock is grantable or not */
+static pl_inode_lock_t *
+__inodelk_grantable (pl_dom_list_t *dom, pl_inode_lock_t *lock)
+{
+        pl_inode_lock_t *l = NULL;
+        pl_inode_lock_t *ret = NULL;
+        if (list_empty (&dom->inodelk_list))
+                goto out;
+        list_for_each_entry (l, &dom->inodelk_list, list){
+                if (inodelk_conflict (lock, l) &&
+                    !same_inodelk_owner (lock, l)) {
+                        ret = l;
+                        goto out;
+                }
+        }
+out:
+        return ret;
+}
+
+static pl_inode_lock_t *
+__blocked_lock_conflict (pl_dom_list_t *dom, pl_inode_lock_t *lock)
+{
+        pl_inode_lock_t *l   = NULL;
+        pl_inode_lock_t *ret = NULL;
+
+        if (list_empty (&dom->blocked_inodelks))
+                return NULL;
+
+        list_for_each_entry (l, &dom->blocked_inodelks, blocked_locks) {
+                if (inodelk_conflict (lock, l)) {
+                        ret = l;
+                        goto out;
+                }
+        }
+
+out:
+        return ret;
+}
+
+static int
+__owner_has_lock (pl_dom_list_t *dom, pl_inode_lock_t *newlock)
+{
+        pl_inode_lock_t *lock = NULL;
+
+        list_for_each_entry (lock, &dom->inodelk_list, list) {
+                if (same_inodelk_owner (lock, newlock))
+                        return 1;
+        }
+
+        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+                if (same_inodelk_owner (lock, newlock))
+                        return 1;
+        }
+
+        return 0;
+}
+
+
+/* Determines if lock can be granted and adds the lock. If the lock
+ * is blocking, adds it to the blocked_inodelks list of the domain.
+ */
+static int
+__lock_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
+                int can_block,  pl_dom_list_t *dom)
+{
+        pl_inode_lock_t *conf = NULL;
+        int ret = -EINVAL;
+
+        conf = __inodelk_grantable (dom, lock);
+        if (conf){
+                ret = -EAGAIN;
+                if (can_block == 0)
+                        goto out;
+
+                gettimeofday (&lock->blkd_time, NULL);
+                list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
+                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                        lock->client_pid,
+                        lkowner_utoa (&lock->owner),
+                        lock->user_flock.l_start,
+                        lock->user_flock.l_len);
+
+
+                goto out;
+        }
+
+        if (__blocked_lock_conflict (dom, lock) && !(__owner_has_lock (dom, lock))) {
+                ret = -EAGAIN;
+                if (can_block == 0)
+                        goto out;
+
+                gettimeofday (&lock->blkd_time, NULL);
+                list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Lock is grantable, but blocking to prevent starvation");
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Blocked",
+                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                        lock->client_pid,
+                        lkowner_utoa (&lock->owner),
+                        lock->user_flock.l_start,
+                        lock->user_flock.l_len);
+
+
+                goto out;
+        }
+        __pl_inodelk_ref (lock);
+        gettimeofday (&lock->granted_time, NULL);
+        list_add (&lock->list, &dom->inodelk_list);
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+/* Return true if the two inodelks have exactly same lock boundaries */
+static int
+inodelks_equal (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+{
+        if ((l1->fl_start == l2->fl_start) &&
+            (l1->fl_end == l2->fl_end))
+                return 1;
+
+        return 0;
+}
+
+
+static pl_inode_lock_t *
+find_matching_inodelk (pl_inode_lock_t *lock, pl_dom_list_t *dom)
+{
+        pl_inode_lock_t *l = NULL;
+        list_for_each_entry (l, &dom->inodelk_list, list) {
+                if (inodelks_equal (l, lock) &&
+                    same_inodelk_owner (l, lock))
+                        return l;
+        }
+        return NULL;
+}
+
+/* Set F_UNLCK removes a lock which has the exact same lock boundaries
+ * as the UNLCK lock specifies. If such a lock is not found, returns invalid
+ */
+static pl_inode_lock_t *
+__inode_unlock_lock (xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
+{
+
+        pl_inode_lock_t *conf = NULL;
+
+        conf = find_matching_inodelk (lock, dom);
+        if (!conf) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        " Matching lock not found for unlock %llu-%llu, by %s "
+                        "on %p", (unsigned long long)lock->fl_start,
+                        (unsigned long long)lock->fl_end,
+                        lkowner_utoa (&lock->owner), lock->client);
+                goto out;
+        }
+        __delete_inode_lock (conf);
+        gf_log (this->name, GF_LOG_DEBUG,
+                " Matching lock found for unlock %llu-%llu, by %s on %p",
+                (unsigned long long)lock->fl_start,
+                (unsigned long long)lock->fl_end, lkowner_utoa (&lock->owner),
+                lock->client);
+
+out:
+        return conf;
+}
+static void
+__grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                             struct list_head *granted, pl_dom_list_t *dom)
+{
+        int              bl_ret = 0;
+        pl_inode_lock_t *bl = NULL;
+        pl_inode_lock_t *tmp = NULL;
+
+        struct list_head blocked_list;
+
+        INIT_LIST_HEAD (&blocked_list);
+        list_splice_init (&dom->blocked_inodelks, &blocked_list);
+
+        list_for_each_entry_safe (bl, tmp, &blocked_list, blocked_locks) {
+
+                list_del_init (&bl->blocked_locks);
+
+                bl_ret = __lock_inodelk (this, pl_inode, bl, 1, dom);
+
+                if (bl_ret == 0) {
+                        list_add (&bl->blocked_locks, granted);
+                }
+        }
+        return;
+}
+
+/* Grant all inodelks blocked on a lock */
+void
+grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_dom_list_t *dom)
+{
+        struct list_head granted;
+        pl_inode_lock_t *lock;
+        pl_inode_lock_t *tmp;
+
+        INIT_LIST_HEAD (&granted);
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __grant_blocked_inode_locks (this, pl_inode, &granted, dom);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Granted",
+                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                        lock->client_pid,
+                        lkowner_utoa (&lock->owner),
+                        lock->user_flock.l_start,
+                        lock->user_flock.l_len);
+
+                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
+                              &lock->user_flock, 0, 0, lock->volume);
+
+                STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0, NULL);
+        }
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
+                        list_del_init (&lock->blocked_locks);
+                        __pl_inodelk_unref (lock);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+}
+
+/* Release all inodelks from this client */
+static int
+release_inode_locks_of_client (xlator_t *this, pl_dom_list_t *dom,
+                                  inode_t *inode, client_t *client)
+{
+        pl_inode_lock_t *tmp = NULL;
+        pl_inode_lock_t *l = NULL;
+
+        pl_inode_t * pinode = NULL;
+
+        struct list_head released;
+
+        char *path = NULL;
+        char *file = NULL;
+
+        INIT_LIST_HEAD (&released);
+
+        pinode = pl_inode_get (this, inode);
+
+        pthread_mutex_lock (&pinode->mutex);
+        {
+
+                list_for_each_entry_safe (l, tmp, &dom->blocked_inodelks, blocked_locks) {
+                        if (l->client != client)
+                                continue;
+
+                        list_del_init (&l->blocked_locks);
+
+                        inode_path (inode, NULL, &path);
+                        if (path)
+                                file = path;
+                        else
+                                file = uuid_utoa (inode->gfid);
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "releasing blocking lock on %s held by "
+                                "{client=%p, pid=%"PRId64" lk-owner=%s}",
+                                file, client, (uint64_t) l->client_pid,
+                                lkowner_utoa (&l->owner));
+
+                        list_add (&l->blocked_locks, &released);
+                        if (path) {
+                                GF_FREE (path);
+                                path = NULL;
+                        }
+                }
+
+                list_for_each_entry_safe (l, tmp, &dom->inodelk_list, list) {
+                        if (l->client != client)
+                                continue;
+
+                        inode_path (inode, NULL, &path);
+                        if (path)
+                                file = path;
+                        else
+                                file = uuid_utoa (inode->gfid);
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "releasing granted lock on %s held by "
+                                "{client=%p, pid=%"PRId64" lk-owner=%s}",
+                                file, client, (uint64_t) l->client_pid,
+                                lkowner_utoa (&l->owner));
+
+                        if (path) {
+                                GF_FREE (path);
+                                path = NULL;
+                        }
+
+                        __delete_inode_lock (l);
+                        __pl_inodelk_unref (l);
+                }
+        }
+        GF_FREE (path);
+
+        pthread_mutex_unlock (&pinode->mutex);
+
+        list_for_each_entry_safe (l, tmp, &released, blocked_locks) {
+                list_del_init (&l->blocked_locks);
+
+                STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN, NULL);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (l);
+        }
+
+        grant_blocked_inode_locks (this, pinode, dom);
+        return 0;
+}
+
+
+static int
+pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
+                int can_block,  pl_dom_list_t *dom)
+{
+        int ret = -EINVAL;
+        pl_inode_lock_t *retlock = NULL;
+        gf_boolean_t    unref = _gf_true;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                if (lock->fl_type != F_UNLCK) {
+                        ret = __lock_inodelk (this, pl_inode, lock, can_block, dom);
+                        if (ret == 0) {
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
+                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                        lock->client_pid,
+                                        lkowner_utoa (&lock->owner),
+                                        lock->fl_start,
+                                        lock->fl_end);
+                        } else if (ret == -EAGAIN) {
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
+                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                        lock->client_pid,
+                                        lkowner_utoa (&lock->owner),
+                                        lock->user_flock.l_start,
+                                        lock->user_flock.l_len);
+                                if (can_block)
+                                        unref = _gf_false;
+                        }
+                } else {
+                        retlock = __inode_unlock_lock (this, lock, dom);
+                        if (!retlock) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "Bad Unlock issued on Inode lock");
+                                ret = -EINVAL;
+                                goto out;
+                        }
+                        __pl_inodelk_unref (retlock);
+
+                        ret = 0;
+                }
+        }
+out:
+        if (unref)
+                __pl_inodelk_unref (lock);
+        pthread_mutex_unlock (&pl_inode->mutex);
+        grant_blocked_inode_locks (this, pl_inode, dom);
+        return ret;
+}
+
+/* Create a new inode_lock_t */
+pl_inode_lock_t *
+new_inode_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
+                call_frame_t *frame, xlator_t *this, const char *volume,
+                char *conn_id)
+
+{
+        pl_inode_lock_t *lock = NULL;
+
+        lock = GF_CALLOC (1, sizeof (*lock),
+                          gf_locks_mt_pl_inode_lock_t);
+        if (!lock) {
+                return NULL;
+        }
+
+        lock->fl_start = flock->l_start;
+        lock->fl_type  = flock->l_type;
+
+        if (flock->l_len == 0)
+                lock->fl_end = LLONG_MAX;
+        else
+                lock->fl_end = flock->l_start + flock->l_len - 1;
+
+        lock->client     = client;
+        lock->client_pid = client_pid;
+        lock->volume     = volume;
+        lock->owner      = frame->root->lk_owner;
+        lock->frame      = frame;
+        lock->this       = this;
+
+        if (conn_id) {
+                lock->connection_id = gf_strdup (conn_id);
+        }
+
+        INIT_LIST_HEAD (&lock->list);
+        INIT_LIST_HEAD (&lock->blocked_locks);
+        __pl_inodelk_ref (lock);
+
+        return lock;
+}
+
+int32_t
+_pl_convert_volume (const char *volume, char **res)
+{
+        char    *mdata_vol = NULL;
+        int     ret = 0;
+
+        mdata_vol = strrchr (volume, ':');
+        //if the volume already ends with :metadata don't bother
+        if (mdata_vol && (strcmp (mdata_vol, ":metadata") == 0))
+                return 0;
+
+        ret = gf_asprintf (res, "%s:metadata", volume);
+        if (ret <= 0)
+                return ENOMEM;
+        return 0;
+}
+
+int32_t
+_pl_convert_volume_for_special_range (struct gf_flock *flock,
+                                      const char *volume, char **res)
+{
+        int32_t     ret = 0;
+
+        if ((flock->l_start == LLONG_MAX -1) &&
+            (flock->l_len == 0)) {
+                ret = _pl_convert_volume (volume, res);
+        }
+
+        return ret;
+}
+
+/* Common inodelk code called from pl_inodelk and pl_finodelk */
+int
+pl_common_inodelk (call_frame_t *frame, xlator_t *this,
+                   const char *volume, inode_t *inode, int32_t cmd,
+                   struct gf_flock *flock, loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+        int32_t           op_ret     = -1;
+        int32_t           op_errno   = 0;
+        int               ret        = -1;
+        GF_UNUSED int     dict_ret   = -1;
+        int               can_block  = 0;
+        pl_inode_t *      pinode     = NULL;
+        pl_inode_lock_t * reqlock    = NULL;
+        pl_dom_list_t *   dom        = NULL;
+        char             *res        = NULL;
+        char             *res1       = NULL;
+        char             *conn_id    = NULL;
+        pl_ctx_t         *ctx        = NULL;
+
+        if (xdata)
+                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (inode, unwind);
+        VALIDATE_OR_GOTO (flock, unwind);
+
+        if ((flock->l_start < 0) || (flock->l_len < 0)) {
+                op_errno = EINVAL;
+                goto unwind;
+        }
+
+        op_errno = _pl_convert_volume_for_special_range (flock, volume, &res);
+        if (op_errno)
+                goto unwind;
+        if (res)
+                volume = res;
+
+        pl_trace_in (this, frame, fd, loc, cmd, flock, volume);
+
+        pinode = pl_inode_get (this, inode);
+        if (!pinode) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        dom = get_domain (pinode, volume);
+        if (!dom) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        if (frame->root->lk_owner.len == 0) {
+                /*
+                  special case: this means release all locks
+                  from this client
+                */
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Releasing all locks from client %p", frame->root->client);
+
+                release_inode_locks_of_client (this, dom, inode, frame->root->client);
+                _pl_convert_volume (volume, &res1);
+                if (res1) {
+                        dom = get_domain (pinode, res1);
+                        if (dom)
+                                release_inode_locks_of_client (this, dom,
+                                                        inode, frame->root->client);
+                }
+
+                op_ret = 0;
+                goto unwind;
+        }
+
+        reqlock = new_inode_lock (flock, frame->root->client, frame->root->pid,
+                                  frame, this, volume, conn_id);
+
+        if (!reqlock) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+
+        switch (cmd) {
+        case F_SETLKW:
+                can_block = 1;
+
+                /* fall through */
+
+        case F_SETLK:
+                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
+                ret = pl_inode_setlk (this, pinode, reqlock,
+                                      can_block, dom);
+
+                if (ret < 0) {
+                        if ((can_block) && (F_UNLCK != flock->l_type)) {
+                                pl_trace_block (this, frame, fd, loc,
+                                                cmd, flock, volume);
+                                goto out;
+                        }
+                        gf_log (this->name, GF_LOG_TRACE, "returning EAGAIN");
+                        op_errno = -ret;
+                        goto unwind;
+                }
+                break;
+
+        default:
+                op_errno = ENOTSUP;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Lock command F_GETLK not supported for [f]inodelk "
+                        "(cmd=%d)",
+                        cmd);
+                goto unwind;
+        }
+
+        op_ret = 0;
+
+        ctx = pl_ctx_get (frame->root->client, this);
+
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+                goto unwind;
+        }
+
+        if (flock->l_type == F_UNLCK)
+                pl_del_locker (ctx->ltable, volume, loc, fd,
+                               &frame->root->lk_owner,
+                               GF_FOP_INODELK);
+        else
+                pl_add_locker (ctx->ltable, volume, loc, fd,
+                               frame->root->pid,
+                               &frame->root->lk_owner,
+                               GF_FOP_INODELK);
+
+unwind:
+        if ((inode != NULL) && (flock !=NULL)) {
+                pl_update_refkeeper (this, inode);
+                pl_trace_out (this, frame, fd, loc, cmd, flock, op_ret, op_errno, volume);
+        }
+
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, NULL);
+out:
+        GF_FREE (res);
+        GF_FREE (res1);
+        return 0;
+}
+
+int
+pl_inodelk (call_frame_t *frame, xlator_t *this,
+            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
+            dict_t *xdata)
+{
+        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock,
+                           loc, NULL, xdata);
+
+        return 0;
+}
+
+int
+pl_finodelk (call_frame_t *frame, xlator_t *this,
+             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+             dict_t *xdata)
+{
+        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock,
+                           NULL, fd, xdata);
+
+        return 0;
+
+}
+
+static inline int32_t
+__get_inodelk_dom_count (pl_dom_list_t *dom)
+{
+        pl_inode_lock_t     *lock   = NULL;
+        int32_t             count   = 0;
+
+        list_for_each_entry (lock, &dom->inodelk_list, list) {
+                count++;
+        }
+        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+                count++;
+        }
+        return count;
+}
+
+/* Returns the no. of locks (blocked/granted) held on a given domain name
+ * If @domname is NULL, returns the no. of locks in all the domains present.
+ * If @domname is non-NULL and non-existent, returns 0 */
+int32_t
+__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname)
+{
+        int32_t            count  = 0;
+        pl_dom_list_t     *dom    = NULL;
+
+        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+                if (domname) {
+                        if (strcmp (domname, dom->domain) == 0) {
+                                count = __get_inodelk_dom_count (dom);
+                                goto out;
+                        }
+
+                } else {
+                    /* Counting locks from all domains */
+                        count += __get_inodelk_dom_count (dom);
+
+                }
+        }
+
+out:
+        return count;
+}
+
+int32_t
+get_inodelk_count (xlator_t *this, inode_t *inode, char *domname)
+{
+        pl_inode_t   *pl_inode = NULL;
+        uint64_t      tmp_pl_inode = 0;
+        int           ret      = 0;
+        int32_t       count    = 0;
+
+        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
+        if (ret != 0) {
+                goto out;
+        }
+
+        pl_inode = (pl_inode_t *)(long) tmp_pl_inode;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                count = __get_inodelk_count (this, pl_inode, domname);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+out:
+        return count;
+}
diff --git a/xlators/features/locks/src/internal.c b/xlators/features/locks/src/internal.c
deleted file mode 100644
index 7f454a78e..000000000
--- a/xlators/features/locks/src/internal.c
+++ /dev/null
@@ -1,762 +0,0 @@
-/*
-  Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "list.h"
-
-#include "locks.h"
-#include "common.h"
-
-
-
-static int
-delete_locks_of_transport (pl_inode_t *pinode, transport_t *trans)
-{
-	posix_lock_t *tmp = NULL;
-	posix_lock_t *l = NULL;
-
-	list_for_each_entry_safe (l, tmp, &pinode->dir_list, list) {
-		if (l->transport == trans) {
-			__delete_lock (pinode, tmp);
-			__destroy_lock (tmp);
-		}
-	}
-
-	return 0;
-}
-
-
-static posix_lock_t *
-__find_exact_matching_lock (pl_inode_t *pinode, posix_lock_t *lock)
-{
-	posix_lock_t *l     = NULL;
-	posix_lock_t *match = NULL;
-
-	list_for_each_entry (l, DOMAIN_HEAD (pinode, GF_LOCK_INTERNAL), list) {
-		if (same_owner (l, lock)
-		    && (l->fl_start == lock->fl_start)
-		    && (l->fl_end   == lock->fl_end)) {
-			match = l;
-			break;
-		}
-	}
-
-	return match;
-}
-
-/**
- * pl_inodelk: 
- *
- * This fop provides fcntl-style locking on files for internal
- * purposes. Locks held through this fop reside in a domain different
- * from those held by applications. This fop is for the use of AFR.
- */
-
-
-static int
-pl_inodelk_common (call_frame_t *frame, xlator_t *this,
-		   inode_t *inode, int32_t cmd, struct flock *flock)
-{
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-	int     can_block = 0;
-
-	posix_locks_private_t * priv       = NULL;
-	transport_t *           transport  = NULL;
-	pid_t                   client_pid = -1;
-	pl_inode_t *            pinode     = NULL;
-	
-	posix_lock_t *          reqlock    = NULL;
-	posix_lock_t *          matchlock  = NULL;  /* steady, fire! */
-
-	VALIDATE_OR_GOTO (frame, unwind);
-	VALIDATE_OR_GOTO (inode, unwind);
-	VALIDATE_OR_GOTO (flock, unwind);
-	
-	if ((flock->l_start < 0) || (flock->l_len < 0)) {
-		op_errno = EINVAL;
-		goto unwind;
-	}
-
-	transport  = frame->root->trans;
-	client_pid = frame->root->pid;
-
-	priv = (posix_locks_private_t *) this->private;
-
-	VALIDATE_OR_GOTO (priv, unwind);
-
-	pinode = pl_inode_get (this, inode);
-	if (!pinode) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		op_errno = ENOMEM;
-		goto unwind;
-	}
-
-	if (client_pid == 0) {
-		/* 
-		   special case: this means release all locks 
-		   from this transport
-		*/
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"releasing all locks from transport %p", transport);
-
-		delete_locks_of_transport (pinode, transport);
-		goto unwind;
-	}
-
-	reqlock = new_posix_lock (flock, transport, client_pid);
-	if (!reqlock) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		op_ret = -1;
-		op_errno = ENOMEM;
-		goto unwind;
-	}
-
-	pthread_mutex_lock (&pinode->mutex);
-	{
-		switch (cmd) {
-		case F_SETLKW:
-			can_block = 1;
-			reqlock->frame = frame;
-			reqlock->this  = this;
-			
-			/* fall through */
-			
-		case F_SETLK:
-			memcpy (&reqlock->user_flock, flock, sizeof (struct flock));
-			
-			switch (flock->l_type) {
-				
-			case F_WRLCK:
-				if (!pl_is_lock_grantable (pinode, reqlock, GF_LOCK_INTERNAL)) {
-					if (can_block) {
-						gf_log (this->name, GF_LOG_DEBUG,
-							"%s (pid=%d) %"PRId64" - %"PRId64" => blocked",
-							reqlock->fl_type == F_UNLCK ? "unlock" : "lock",
-							reqlock->client_pid,
-							reqlock->user_flock.l_start,
-							reqlock->user_flock.l_len);
-						pl_insert_lock (pinode, reqlock, GF_LOCK_INTERNAL);
-						
-						goto unlock;
-					}
-					
-					__destroy_lock (reqlock);
-					
-					
-					gf_log (this->name, GF_LOG_DEBUG,
-						"%s (pid=%d) %"PRId64" - %"PRId64" => NOK",
-						reqlock->fl_type == F_UNLCK ? "unlock" : "lock",
-						reqlock->client_pid, reqlock->user_flock.l_start,
-						reqlock->user_flock.l_len);
-					op_errno = EAGAIN;
-					
-					goto unlock;
-				}
-				
-				gf_log (this->name, GF_LOG_DEBUG,
-					"%s (pid=%d) %"PRId64" - %"PRId64" => OK",
-					reqlock->fl_type == F_UNLCK ? "unlock" : "lock",
-					reqlock->client_pid,
-					reqlock->user_flock.l_start,
-					reqlock->user_flock.l_len);
-				pl_insert_lock (pinode, reqlock, GF_LOCK_INTERNAL);
-				
-				break;
-				
-			case F_UNLCK:
-				matchlock = __find_exact_matching_lock (pinode, reqlock);
-
-				__destroy_lock (reqlock);
-				if (!matchlock) {
-					op_errno = EINVAL;
-					goto unlock;
-				}
-				
-				__delete_lock (pinode, matchlock);
-				__destroy_lock (matchlock);
-				
-				break;
-				
-			default:
-				op_errno = ENOTSUP;
-				gf_log (this->name, GF_LOG_ERROR,
-					"lock type %d not supported for [F]INODELK",
-					flock->l_type);
-				goto unlock;
-			}
-			
-			
-			break;
-			
-		default:
-			op_errno = ENOTSUP;
-			gf_log (this->name, GF_LOG_ERROR,
-				"lock command F_GETLK not supported for [F]INODELK (cmd=%d)", 
-				cmd);
-			goto unlock;
-		}
-		
-		op_ret = 0;
-		
-	unlock:
-		if (pinode)
-			pthread_mutex_unlock (&pinode->mutex);
-	}
-
-unwind:
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-int
-pl_inodelk (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t cmd, struct flock *flock)
-{
-	return pl_inodelk_common (frame, this, loc->inode, cmd, flock);
-}
-
-
-int
-pl_finodelk (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, int32_t cmd, struct flock *flock)
-{
-	return pl_inodelk_common (frame, this, fd->inode, cmd, flock);
-}
-
-
-/**
- * types_conflict - do two types of lock conflict?
- * @t1: type
- * @t2: type
- *
- * two read locks do not conflict
- * any other case conflicts
- */
-
-static int
-types_conflict (entrylk_type t1, entrylk_type t2)
-{
-	return !((t1 == ENTRYLK_RDLCK) && (t2 == ENTRYLK_RDLCK));
-}
-
-/**
- * all_names - does a basename represent all names?
- * @basename: name to check
- */
-
-#define all_names(basename) ((basename == NULL) ? 1 : 0)
-
-/**
- * names_conflict - do two names conflict?
- * @n1: name
- * @n2: name
- */
-
-static int 
-names_conflict (const char *n1, const char *n2)
-{
-	return all_names (n1) || all_names (n2) || !strcmp (n1, n2);
-}
-
-
-static int 
-names_equal (const char *n1, const char *n2)
-{
-	return (n1 == NULL && n2 == NULL) || (n1 && n2 && !strcmp (n1, n2));
-}
-
-/**
- * lock_grantable - is this lock grantable?
- * @inode: inode in which to look
- * @basename: name we're trying to lock
- * @type: type of lock
- */
-
-static pl_entry_lock_t *
-__lock_grantable (pl_inode_t *pinode, const char *basename, entrylk_type type)
-{
-	pl_entry_lock_t *lock = NULL;
-
-	if (list_empty (&pinode->dir_list))
-		return NULL;
-
-	list_for_each_entry (lock, &pinode->dir_list, inode_list) {
-		if (names_conflict (lock->basename, basename) &&
-		    types_conflict (lock->type, type))
-			return lock;
-	}
-
-	return NULL;
-}
-
-/**
- * find_most_matching_lock - find the lock struct which most matches in order of:
- *                           lock on the exact basename ||
- *                           an all_names lock
- *                      
- *
- * @inode: inode in which to look
- * @basename: name to search for
- */
-
-static pl_entry_lock_t * 
-__find_most_matching_lock (pl_inode_t *pinode, const char *basename)
-{
-	pl_entry_lock_t *lock;
-	pl_entry_lock_t *all = NULL;
-	pl_entry_lock_t *exact = NULL;
-
-	if (list_empty (&pinode->dir_list)) 
-		return NULL;
-
-	list_for_each_entry (lock, &pinode->dir_list, inode_list) {
-		if (all_names (lock->basename))
-			all = lock;
-		else if (names_equal (lock->basename, basename))
-			exact = lock;
-	}
-
-	return (exact ? exact : all);
-}
-
-
-/**
- * insert_new_lock - insert a new dir lock into the inode with the given parameters
- * @pinode: inode to insert into
- * @basename: basename for the lock
- * @type: type of the lock
- */
-
-static pl_entry_lock_t *
-new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
-		  transport_t *trans)
-{
-	pl_entry_lock_t *newlock = NULL;
-
-	newlock = CALLOC (sizeof (pl_entry_lock_t), 1);
-	if (!newlock) {
-		goto out;
-	}
-
-	newlock->basename = basename ? strdup (basename) : NULL;
-	newlock->type     = type;
-	newlock->trans    = trans;
-
-	if (type == ENTRYLK_RDLCK)
-		newlock->read_count = 1;
-
-	INIT_LIST_HEAD (&newlock->inode_list);
-	INIT_LIST_HEAD (&newlock->blocked_locks);
-
-out:
-	return newlock;
-}
-
-/**
- * lock_name - lock a name in a directory
- * @inode: inode for the directory in which to lock
- * @basename: name of the entry to lock
- *            if null, lock the entire directory
- *            
- * the entire directory being locked is represented as: a single
- * pl_entry_lock_t present in the entrylk_locks list with its
- * basename = NULL
- */
-
-int
-__lock_name (pl_inode_t *pinode, const char *basename, entrylk_type type,
-	     call_frame_t *frame, xlator_t *this, int nonblock)
-{
-	pl_entry_lock_t *lock    = NULL;
-	pl_entry_lock_t *conf    = NULL;
-
-	transport_t *trans = NULL;
-
-	int ret = -EINVAL;
-
-	trans = frame->root->trans;
-
-	conf = __lock_grantable (pinode, basename, type);
-	if (conf) {
-		ret = -EAGAIN;
-		if (nonblock)
-			goto out;
-
-		lock = new_entrylk_lock (pinode, basename, type, trans);
-
-		if (!lock) {
-			ret = -ENOMEM;
-			goto out;
-		}
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"blocking lock: {pinode=%p, basename=%s}",
-			pinode, basename);
-
-		lock->frame   = frame;
-		lock->this    = this;
-		lock->blocked = 1;
-
-		list_add (&lock->blocked_locks, &conf->blocked_locks);
-
-
-		goto out;
-	}
-		
-	switch (type) {
-	case ENTRYLK_RDLCK:
-		lock = __find_most_matching_lock (pinode, basename);
-
-		if (lock && names_equal (lock->basename, basename)) {
-			lock->read_count++;
-
-			FREE (lock->basename);
-			FREE (lock);
-
-			lock = NULL;
-		} else {
-			lock = new_entrylk_lock (pinode, basename, type, trans);
-
-			if (!lock) {
-				ret = -ENOMEM;
-				goto out;
-			}
-
-			list_add (&lock->inode_list, &pinode->dir_list);
-		}
-		break;
-
-	case ENTRYLK_WRLCK:
-		lock = new_entrylk_lock (pinode, basename, type, trans);
-			
-		if (!lock) {
-			ret = -ENOMEM;
-			goto out;
-		}
-
-		list_add (&lock->inode_list, &pinode->dir_list);
-		break;
-	}
-
-	ret = 0;
-out:
-	return ret;
-}
-
-
-/**
- * unlock_name - unlock a name in a directory
- * @inode: inode for the directory to unlock in
- * @basename: name of the entry to unlock
- *            if null, unlock the entire directory
- */
-
-pl_entry_lock_t *
-__unlock_name (pl_inode_t *pinode, const char *basename, entrylk_type type)
-{
-	pl_entry_lock_t *lock = NULL;
-	pl_entry_lock_t *ret_lock = NULL;
-
-	lock = __find_most_matching_lock (pinode, basename);
-	
-	if (!lock) {
-		gf_log ("locks", GF_LOG_DEBUG,
-			"unlock on %s (type=%s) attempted but no matching lock found",
-			basename, type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" : 
-			"ENTRYLK_WRLCK");
-		goto out;
-	}
-	
-	if (names_equal (lock->basename, basename)
-	    && lock->type == type) {
-		if (type == ENTRYLK_RDLCK) {
-			lock->read_count--;
-		}
-		if (type == ENTRYLK_WRLCK || lock->read_count == 0) {
-			list_del (&lock->inode_list);
-			ret_lock = lock;
-		}
-	} else {
-		gf_log ("locks", GF_LOG_ERROR,
-			"unlock for a non-existing lock!");
-		goto out;
-	}
-
-out:
-	return ret_lock;
-}
-
-
-void
-__grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-			     pl_entry_lock_t *lock,
-			     struct list_head *granted)
-{
-	int              bl_ret = 0;
-	pl_entry_lock_t *bl   = NULL;
-	pl_entry_lock_t *tmp  = NULL;
-
-	list_for_each_entry_safe (bl, tmp, &lock->blocked_locks, 
-				  blocked_locks) {
-		list_del_init (&bl->blocked_locks);
-
-		/* TODO: error checking */
-
-		gf_log ("locks", GF_LOG_DEBUG,
-			"trying to unblock: {pinode=%p, basename=%s}",
-			pl_inode, bl->basename);
-					
-		bl_ret = __lock_name (pl_inode, bl->basename, bl->type,
-				      bl->frame, bl->this, 0);
-
-		if (bl_ret == 0) {
-			list_add (&bl->blocked_locks, granted);
-		} else {
-			if (bl->basename)
-				FREE (bl->basename);
-			FREE (bl);
-		}
-	}	
-	return;
-}
-
-
-void
-grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-			   pl_entry_lock_t *unlocked)
-{
-	struct list_head  granted_list;
-	pl_entry_lock_t  *tmp = NULL;
-	pl_entry_lock_t  *lock = NULL;
-
-	INIT_LIST_HEAD (&granted_list);
-
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
-		__grant_blocked_entry_locks (this, pl_inode, unlocked,
-					     &granted_list);
-	}
-	pthread_mutex_unlock (&pl_inode->mutex);
-
-	list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
-		list_del_init (&lock->blocked_locks);
-
-		STACK_UNWIND (lock->frame, 0, 0);
-
-		FREE (lock->basename);
-		FREE (lock);
-	}
-
-	FREE (unlocked->basename);
-	FREE (unlocked);
-
-	return;
-}
-
-
-/**
- * release_entry_locks_for_transport: release all entry locks from this
- * transport for this loc_t
- */
-
-static int
-release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
-				   transport_t *trans)
-{
-	pl_entry_lock_t  *lock;
-	pl_entry_lock_t  *tmp;
-	struct list_head  granted;
-
-	INIT_LIST_HEAD (&granted);
-
-	pthread_mutex_lock (&pinode->mutex);
-	{
-		if (list_empty (&pinode->dir_list)) {
-			goto unlock;
-		}
-
-		list_for_each_entry_safe (lock, tmp, &pinode->dir_list,
-					  inode_list) {
-			if (lock->trans != trans)
-				continue;
-
-			list_del_init (&lock->inode_list);
-			__grant_blocked_entry_locks (this, pinode, lock,
-						     &granted);
-
-			FREE (lock->basename);
-			FREE (lock);
-		}
-	}
-unlock:
-	pthread_mutex_unlock (&pinode->mutex);
-
-	list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
-		list_del_init (&lock->blocked_locks);
-
-		STACK_UNWIND (lock->frame, 0, 0);
-
-		FREE (lock->basename);
-		FREE (lock);
-	}
-
-	return 0;
-}
-
-
-/**
- * pl_entrylk:
- * 
- * Locking on names (directory entries)
- */
-
-int
-pl_entrylk_common (call_frame_t *frame, xlator_t *this,
-		   inode_t *inode, const char *basename, 
-		   entrylk_cmd cmd, entrylk_type type)
-{
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	transport_t * transport = NULL;
-	pid_t pid = -1;
-
-	pl_inode_t *       pinode = NULL; 
-	int                ret    = -1;
-	pl_entry_lock_t   *unlocked = NULL;
-	char               unwind = 1;
-
-	pinode = pl_inode_get (this, inode);
-	if (!pinode) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-	pid       = frame->root->pid;
-	transport = frame->root->trans;
-
-	if (pid == 0) {
-		/* 
-		   this is a special case that means release
-		   all locks from this transport 
-		*/
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"releasing locks for transport %p", transport);
-
-		release_entry_locks_for_transport (this, pinode, transport);
-		op_ret = 0;
-
-		goto out;
-	}
-
-	switch (cmd) {
-	case ENTRYLK_LOCK:
-		pthread_mutex_lock (&pinode->mutex);
-		{
-			ret = __lock_name (pinode, basename, type,
-					   frame, this, 0);
-		}
-		pthread_mutex_unlock (&pinode->mutex);
-
-		if (ret < 0) {
-			if (ret == -EAGAIN)
-				unwind = 0;
-			op_errno = -ret;
-			goto out;
-		}
-
-		break;
-
-	case ENTRYLK_LOCK_NB:
-		pthread_mutex_lock (&pinode->mutex);
-		{
-			ret = __lock_name (pinode, basename, type,
-					   frame, this, 1);
-		}
-		pthread_mutex_unlock (&pinode->mutex);
-
-		if (ret < 0) {
-			op_errno = -ret;
-			goto out;
-		}
-
-		break;
-
-	case ENTRYLK_UNLOCK:
-		pthread_mutex_lock (&pinode->mutex);
-		{
-			unlocked = __unlock_name (pinode, basename, type);
-		}
-		pthread_mutex_unlock (&pinode->mutex);
-
-		if (unlocked)
-			grant_blocked_entry_locks (this, pinode, unlocked);
-
-		break;
-
-	default:
-		gf_log (this->name, GF_LOG_ERROR,
-			"unexpected case!");
-		goto out;
-	}
-
-	op_ret = 0;
-out:
-	if (unwind) {
-		STACK_UNWIND (frame, op_ret, op_errno);
-	}
-	
-	return 0;
-}
-
-
-int
-pl_entrylk (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, const char *basename, 
-	    entrylk_cmd cmd, entrylk_type type)
-{
-	return pl_entrylk_common (frame, this, loc->inode, basename, cmd, type);
-}
-
-
-int
-pl_fentrylk (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, const char *basename, 
-	     entrylk_cmd cmd, entrylk_type type)
-{
-	return pl_entrylk_common (frame, this, fd->inode, basename, cmd, type);	
-}
diff --git a/xlators/features/locks/src/locks-mem-types.h b/xlators/features/locks/src/locks-mem-types.h
new file mode 100644
index 000000000..08aeb0a79
--- /dev/null
+++ b/xlators/features/locks/src/locks-mem-types.h
@@ -0,0 +1,29 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __LOCKS_MEM_TYPES_H__
+#define __LOCKS_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_locks_mem_types_ {
+        gf_locks_mt_pl_dom_list_t = gf_common_mt_end + 1,
+        gf_locks_mt_pl_inode_t,
+        gf_locks_mt_posix_lock_t,
+        gf_locks_mt_pl_entry_lock_t,
+        gf_locks_mt_pl_inode_lock_t,
+        gf_locks_mt_truncate_ops,
+        gf_locks_mt_pl_rw_req_t,
+        gf_locks_mt_posix_locks_private_t,
+        gf_locks_mt_pl_fdctx_t,
+        gf_locks_mt_end
+};
+#endif
+
diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h
index 8ed7bb63f..76fc941d7 100644
--- a/xlators/features/locks/src/locks.h
+++ b/xlators/features/locks/src/locks.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __POSIX_LOCKS_H__
 #define __POSIX_LOCKS_H__
 
@@ -26,86 +16,177 @@
 #endif
 
 #include "compat-errno.h"
-#include "transport.h"
 #include "stack.h"
 #include "call-stub.h"
+#include "locks-mem-types.h"
+#include "client_t.h"
+
+#include "lkowner.h"
 
 struct __pl_fd;
 
 struct __posix_lock {
-	struct list_head   list;
+        struct list_head   list;
+
+        short              fl_type;
+        off_t              fl_start;
+        off_t              fl_end;
 
-	short              fl_type;
-	off_t              fl_start;
-	off_t              fl_end;
+        short              blocked;    /* waiting to acquire */
+        struct gf_flock    user_flock; /* the flock supplied by the user */
+        xlator_t          *this;       /* required for blocked locks */
+        unsigned long      fd_num;
 
-	short              blocked;    /* waiting to acquire */
-	struct flock       user_flock; /* the flock supplied by the user */
-	xlator_t          *this;       /* required for blocked locks */
-	fd_t              *fd;
+        fd_t              *fd;
+        call_frame_t      *frame;
 
-	call_frame_t      *frame;
+        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
+        struct timeval     granted_time; /*time at which lock was queued into active list*/
 
-	/* These two together serve to uniquely identify each process
-	   across nodes */
+        /* These two together serve to uniquely identify each process
+           across nodes */
 
-	transport_t       *transport;     /* to identify client node */
-	pid_t              client_pid;    /* pid of client process */
+        void              *client;     /* to identify client node */
+        gf_lkowner_t       owner;
+        pid_t              client_pid;    /* pid of client process */
 };
 typedef struct __posix_lock posix_lock_t;
 
+struct __pl_inode_lock {
+        struct list_head   list;
+        struct list_head   blocked_locks; /* list_head pointing to blocked_inodelks */
+        int                ref;
+
+        short              fl_type;
+        off_t              fl_start;
+        off_t              fl_end;
+
+        const char        *volume;
+
+        struct gf_flock    user_flock; /* the flock supplied by the user */
+        xlator_t          *this;       /* required for blocked locks */
+        fd_t              *fd;
+
+        call_frame_t      *frame;
+
+        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
+        struct timeval     granted_time; /*time at which lock was queued into active list*/
+
+        /* These two together serve to uniquely identify each process
+           across nodes */
+
+        void              *client;     /* to identify client node */
+        gf_lkowner_t       owner;
+        pid_t              client_pid;    /* pid of client process */
+
+        char              *connection_id; /* stores the client connection id */
+};
+typedef struct __pl_inode_lock pl_inode_lock_t;
+
 struct __pl_rw_req_t {
-	struct list_head      list;
-	call_stub_t          *stub;
-	posix_lock_t          region;
+        struct list_head      list;
+        call_stub_t          *stub;
+        posix_lock_t          region;
 };
 typedef struct __pl_rw_req_t pl_rw_req_t;
 
+struct __pl_dom_list_t {
+        struct list_head   inode_list;       /* list_head back to pl_inode_t */
+        const char        *domain;
+        struct list_head   entrylk_list;     /* List of entry locks */
+        struct list_head   blocked_entrylks; /* List of all blocked entrylks */
+        struct list_head   inodelk_list;     /* List of inode locks */
+        struct list_head   blocked_inodelks; /* List of all blocked inodelks */
+};
+typedef struct __pl_dom_list_t pl_dom_list_t;
 
 struct __entry_lock {
-	struct list_head  inode_list;    /* list_head back to pl_inode_t */
-	struct list_head  blocked_locks; /* locks blocked due to this lock */
-
-	call_frame_t     *frame;
-	xlator_t         *this;
-	int               blocked;
-	
-	const char       *basename;
-	entrylk_type      type;
-	unsigned int      read_count;    /* number of read locks */
-	transport_t      *trans;
+        struct list_head  domain_list;    /* list_head back to pl_dom_list_t */
+        struct list_head  blocked_locks; /* list_head back to blocked_entrylks */
+
+        call_frame_t     *frame;
+        xlator_t         *this;
+
+        const char       *volume;
+
+        const char       *basename;
+        entrylk_type      type;
+
+        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
+        struct timeval     granted_time; /*time at which lock was queued into active list*/
+
+        void             *trans;
+        gf_lkowner_t      owner;
+	pid_t             client_pid;    /* pid of client process */
+
+        char             *connection_id; /* stores the client connection id */
 };
 typedef struct __entry_lock pl_entry_lock_t;
 
 
-/* The "simulated" inode. This contains a list of all the locks associated 
+/* The "simulated" inode. This contains a list of all the locks associated
    with this file */
 
 struct __pl_inode {
-	pthread_mutex_t  mutex;
-
-	struct list_head dir_list;       /* list of entry locks */
-	struct list_head ext_list;       /* list of fcntl locks */
-	struct list_head int_list;       /* list of internal locks */
-	struct list_head rw_list;        /* list of waiting r/w requests */
-	int              mandatory;      /* if mandatory locking is enabled */
+        pthread_mutex_t  mutex;
+
+        struct list_head dom_list;       /* list of domains */
+        struct list_head ext_list;       /* list of fcntl locks */
+        struct list_head rw_list;        /* list of waiting r/w requests */
+        struct list_head reservelk_list;        /* list of reservelks */
+        struct list_head blocked_reservelks;        /* list of blocked reservelks */
+        struct list_head blocked_calls;  /* List of blocked lock calls while a reserve is held*/
+        int              mandatory;      /* if mandatory locking is enabled */
+
+        inode_t          *refkeeper;     /* hold refs on an inode while locks are
+                                            held to prevent pruning */
 };
 typedef struct __pl_inode pl_inode_t;
 
-#define DOMAIN_HEAD(pl_inode, dom) (dom == GF_LOCK_POSIX	\
-				    ? &pl_inode->ext_list	\
-				    : &pl_inode->int_list)
-
 
 struct __pl_fd {
-	gf_boolean_t nonblocking;       /* whether O_NONBLOCK has been set */
+        gf_boolean_t nonblocking;       /* whether O_NONBLOCK has been set */
 };
 typedef struct __pl_fd pl_fd_t;
 
 
 typedef struct {
-	gf_boolean_t    mandatory;      /* if mandatory locking is enabled */
+        gf_boolean_t    mandatory;      /* if mandatory locking is enabled */
+        gf_boolean_t    trace;          /* trace lock requests in and out */
+        char           *brickname;
 } posix_locks_private_t;
 
 
+typedef struct {
+        gf_boolean_t   entrylk_count_req;
+        gf_boolean_t   inodelk_count_req;
+        gf_boolean_t   inodelk_dom_count_req;
+        gf_boolean_t   posixlk_count_req;
+        gf_boolean_t   parent_entrylk_req;
+
+        /* used by {f,}truncate */
+        loc_t  loc;
+        fd_t  *fd;
+        off_t  offset;
+        dict_t *xdata;
+        enum {TRUNCATE, FTRUNCATE} op;
+} pl_local_t;
+
+
+typedef struct {
+        struct list_head locks_list;
+} pl_fdctx_t;
+
+
+typedef struct _locks_ctx {
+        gf_lock_t            ltable_lock; /* only for replace, 
+                                             ltable has its own internal 
+                                             lock for operations */
+        struct _lock_table  *ltable;
+} pl_ctx_t;
+
+
+pl_ctx_t *
+pl_ctx_get (client_t *client, xlator_t *xlator);
+
 #endif /* __POSIX_LOCKS_H__ */
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index e2b336607..7bfb38a51 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -36,6 +26,10 @@
 
 #include "locks.h"
 #include "common.h"
+#include "statedump.h"
+#include "clear.h"
+#include "defaults.h"
+#include "syncop.h"
 
 #ifndef LLONG_MAX
 #define LLONG_MAX LONG_LONG_MAX /* compat with old gcc */
@@ -46,789 +40,2733 @@
 
 void do_blocked_rw (pl_inode_t *);
 static int __rw_allowable (pl_inode_t *, posix_lock_t *, glusterfs_fop_t);
+static int format_brickname(char *);
+int pl_lockinfo_get_brickname (xlator_t *, inode_t *, int32_t *);
+static int fetch_pathinfo(xlator_t *, inode_t *, int32_t *, char **);
 
-struct _truncate_ops {
-	loc_t  loc;
-	fd_t  *fd;
-	off_t  offset;
-	enum {TRUNCATE, FTRUNCATE} op;
-};
+static pl_fdctx_t *
+pl_new_fdctx ()
+{
+        pl_fdctx_t *fdctx = NULL;
 
+        fdctx = GF_CALLOC (1, sizeof (*fdctx),
+                           gf_locks_mt_pl_fdctx_t);
+        GF_VALIDATE_OR_GOTO ("posix-locks", fdctx, out);
+
+        INIT_LIST_HEAD (&fdctx->locks_list);
+
+out:
+        return fdctx;
+}
+
+static pl_fdctx_t *
+pl_check_n_create_fdctx (xlator_t *this, fd_t *fd)
+{
+        int         ret   = 0;
+        uint64_t    tmp   = 0;
+        pl_fdctx_t *fdctx = NULL;
+
+        GF_VALIDATE_OR_GOTO ("posix-locks", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+
+        LOCK (&fd->lock);
+        {
+                ret = __fd_ctx_get (fd, this, &tmp);
+                if ((ret != 0) || (tmp == 0)) {
+                        fdctx = pl_new_fdctx ();
+                        if (fdctx == NULL) {
+                                goto unlock;
+                        }
+                }
+
+                ret = __fd_ctx_set (fd, this, (uint64_t)(long)fdctx);
+                if (ret != 0) {
+                        GF_FREE (fdctx);
+                        fdctx = NULL;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "failed to set fd ctx");
+                }
+        }
+unlock:
+        UNLOCK (&fd->lock);
+
+out:
+        return fdctx;
+}
 
 int
 pl_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int32_t op_ret, int32_t op_errno, struct stat *buf)
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-	struct _truncate_ops *local = NULL;
+        pl_local_t *local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	if (local->op == TRUNCATE)
-		loc_wipe (&local->loc);
+        if (local->op == TRUNCATE)
+                loc_wipe (&local->loc);
 
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
+        if (local->xdata)
+                dict_unref (local->xdata);
+        if (local->fd)
+                fd_unref (local->fd);
+
+        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
+                             prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 static int
-truncate_allowed (pl_inode_t *pl_inode, 
-		  transport_t *transport, pid_t client_pid, 
-		  off_t offset)
+truncate_allowed (pl_inode_t *pl_inode,
+                  client_t *client, pid_t client_pid,
+                  gf_lkowner_t *owner, off_t offset)
 {
-	posix_lock_t *l = NULL;
-	posix_lock_t  region = {.list = {0, }, };
-	int           ret = 1;
-
-	region.fl_start   = offset;
-	region.fl_end     = LLONG_MAX;
-	region.transport  = transport;
-	region.client_pid = client_pid;
-
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
-		list_for_each_entry (l, &pl_inode->ext_list, list) {
-			if (!l->blocked
-			    && locks_overlap (&region, l)
-			    && !same_owner (&region, l)) {
-				ret = 0;
-				break;
-			}
-		}
-	}
-	pthread_mutex_unlock (&pl_inode->mutex);
-
-	return ret;
+        posix_lock_t *l = NULL;
+        posix_lock_t  region = {.list = {0, }, };
+        int           ret = 1;
+
+        region.fl_start   = offset;
+        region.fl_end     = LLONG_MAX;
+        region.client     = client;
+        region.client_pid = client_pid;
+        region.owner      = *owner;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (l, &pl_inode->ext_list, list) {
+                        if (!l->blocked
+                            && locks_overlap (&region, l)
+                            && !same_owner (&region, l)) {
+                                ret = 0;
+                                gf_log ("posix-locks", GF_LOG_TRACE, "Truncate "
+                                        "allowed");
+                                break;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        return ret;
 }
 
 
 static int
 truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, struct stat *buf)
-{
-	posix_locks_private_t *priv = NULL;
-	struct _truncate_ops  *local = NULL;
-	inode_t               *inode = NULL;
-	pl_inode_t            *pl_inode = NULL;
-
-
-	priv = this->private;
-	local = frame->local;
-
-	if (op_ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"got error (errno=%d, stderror=%s) from child", 
-			op_errno, strerror (op_errno));
-		goto unwind;
-	}
-
-	if (local->op == TRUNCATE)
-		inode = local->loc.inode;
-	else
-		inode = local->fd->inode;
-
-	pl_inode = pl_inode_get (this, inode);
-	if (!pl_inode) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"unable to get pl_inode from %p", inode);
-		op_errno = ENOMEM;
-		goto unwind;
-	}
-
-	if (priv->mandatory
-	    && pl_inode->mandatory
-	    && !truncate_allowed (pl_inode, frame->root->trans,
-				  frame->root->pid, local->offset)) {
-		op_errno = EAGAIN;
-		goto unwind;
-	}
-
-	switch (local->op) {
-	case TRUNCATE:
-		STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
-			    FIRST_CHILD (this)->fops->truncate,
-			    &local->loc, local->offset);
-		break;
-	case FTRUNCATE:
-		STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
-			    FIRST_CHILD (this)->fops->ftruncate,
-			    local->fd, local->offset);
-		break;
-	}
-
-	return 0;
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
+{
+        posix_locks_private_t *priv = NULL;
+        pl_local_t            *local = NULL;
+        inode_t               *inode = NULL;
+        pl_inode_t            *pl_inode = NULL;
+
+
+        priv = this->private;
+        local = frame->local;
+
+        if (op_ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "got error (errno=%d, stderror=%s) from child",
+                        op_errno, strerror (op_errno));
+                goto unwind;
+        }
+
+        if (local->op == TRUNCATE)
+                inode = local->loc.inode;
+        else
+                inode = local->fd->inode;
+
+        pl_inode = pl_inode_get (this, inode);
+        if (!pl_inode) {
+                op_ret   = -1;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        if (priv->mandatory
+            && pl_inode->mandatory
+            && !truncate_allowed (pl_inode, frame->root->client,
+                                  frame->root->pid, &frame->root->lk_owner,
+                                  local->offset)) {
+                op_ret   = -1;
+                op_errno = EAGAIN;
+                goto unwind;
+        }
+
+        switch (local->op) {
+        case TRUNCATE:
+                STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->truncate,
+                            &local->loc, local->offset, local->xdata);
+                break;
+        case FTRUNCATE:
+                STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->ftruncate,
+                            local->fd, local->offset, local->xdata);
+                break;
+        }
+
+        return 0;
 
 unwind:
-	if (local->op == TRUNCATE)
-		loc_wipe (&local->loc);
-
-	STACK_UNWIND (frame, -1, ENOMEM, buf);
-	return 0;
+        gf_log (this->name, GF_LOG_ERROR, "truncate failed with ret: %d, "
+                "error: %s", op_ret, strerror (op_errno));
+        if (local->op == TRUNCATE)
+                loc_wipe (&local->loc);
+        if (local->xdata)
+                dict_unref (local->xdata);
+        if (local->fd)
+                fd_unref (local->fd);
+
+        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, NULL, xdata);
+        return 0;
 }
 
 
 int
 pl_truncate (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, off_t offset)
+             loc_t *loc, off_t offset, dict_t *xdata)
 {
-	struct _truncate_ops *local = NULL;
+        pl_local_t *local = NULL;
 
-	local = CALLOC (1, sizeof (struct _truncate_ops));
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto unwind;
-	}
+        local = mem_get0 (this->local_pool);
+        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
 
-	local->op         = TRUNCATE;
-	local->offset     = offset;
-	loc_copy (&local->loc, loc);
+        local->op         = TRUNCATE;
+        local->offset     = offset;
+        loc_copy (&local->loc, loc);
+        if (xdata)
+                local->xdata = dict_ref (xdata);
 
-	frame->local = local;
+        frame->local = local;
 
-	STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->stat, loc);
+        STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->stat, loc, NULL);
 
-	return 0;
+        return 0;
 
 unwind:
-	STACK_UNWIND (frame, -1, ENOMEM, NULL);
+        gf_log (this->name, GF_LOG_ERROR, "truncate for %s failed with ret: %d, "
+                "error: %s", loc->path, -1, strerror (ENOMEM));
+        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 pl_ftruncate (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, off_t offset)
+              fd_t *fd, off_t offset, dict_t *xdata)
 {
-	struct _truncate_ops *local = NULL;
+        pl_local_t *local = NULL;
 
-	local = CALLOC (1, sizeof (struct _truncate_ops));
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		goto unwind;
-	}
+        local = mem_get0 (this->local_pool);
+        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
 
-	local->op         = FTRUNCATE;
-	local->offset     = offset;
-	local->fd         = fd;
+        local->op         = FTRUNCATE;
+        local->offset     = offset;
+        local->fd         = fd_ref (fd);
+        if (xdata)
+                local->xdata = dict_ref (xdata);
 
-	frame->local = local;
+        frame->local = local;
 
-	STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fstat, fd);
-	return 0;
+        STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
 
 unwind:
-	STACK_UNWIND (frame, -1, ENOMEM, NULL);
+        gf_log (this->name, GF_LOG_ERROR, "ftruncate failed with ret: %d, "
+                "error: %s", -1, strerror (ENOMEM));
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
+int
+pl_locks_by_fd (pl_inode_t *pl_inode, fd_t *fd)
+{
+       posix_lock_t *l = NULL;
+       int found = 0;
+
+       pthread_mutex_lock (&pl_inode->mutex);
+       {
+
+               list_for_each_entry (l, &pl_inode->ext_list, list) {
+                       if ((l->fd_num == fd_to_fdnum(fd))) {
+                               found = 1;
+                               break;
+                       }
+               }
+
+       }
+       pthread_mutex_unlock (&pl_inode->mutex);
+       return found;
+}
+
+static void
+delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
+{
+       posix_lock_t *tmp = NULL;
+       posix_lock_t *l = NULL;
+
+       struct list_head blocked_list;
+
+       INIT_LIST_HEAD (&blocked_list);
+
+       pthread_mutex_lock (&pl_inode->mutex);
+       {
+
+               list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
+                       if ((l->fd_num == fd_to_fdnum(fd))) {
+                               if (l->blocked) {
+                                       list_move_tail (&l->list, &blocked_list);
+                                       continue;
+                               }
+                               __delete_lock (pl_inode, l);
+                               __destroy_lock (l);
+                       }
+               }
+
+       }
+       pthread_mutex_unlock (&pl_inode->mutex);
+
+       list_for_each_entry_safe (l, tmp, &blocked_list, list) {
+               list_del_init(&l->list);
+               STACK_UNWIND_STRICT (lk, l->frame, -1, EAGAIN, &l->user_flock,
+                                    NULL);
+               __destroy_lock (l);
+       }
+
+        grant_blocked_locks (this, pl_inode);
+
+        do_blocked_rw (pl_inode);
+
+}
 
 static void
 __delete_locks_of_owner (pl_inode_t *pl_inode,
-			 transport_t *transport, pid_t pid)
+                         client_t *client, gf_lkowner_t *owner)
 {
-	posix_lock_t *tmp = NULL;
-	posix_lock_t *l = NULL;
+        posix_lock_t *tmp = NULL;
+        posix_lock_t *l = NULL;
+
+        /* TODO: what if it is a blocked lock with pending l->frame */
+
+        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
+                if (l->blocked)
+                        continue;
+                if ((l->client == client) &&
+                    is_same_lkowner (&l->owner, owner)) {
+                        gf_log ("posix-locks", GF_LOG_TRACE,
+                                " Flushing lock"
+                                "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" state: %s",
+                                l->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                l->client_pid,
+                                lkowner_utoa (&l->owner),
+                                l->user_flock.l_start,
+                                l->user_flock.l_len,
+                                l->blocked == 1 ? "Blocked" : "Active");
+
+                        __delete_lock (pl_inode, l);
+                        __destroy_lock (l);
+                }
+        }
+
+        return;
+}
 
-	/* TODO: what if it is a blocked lock with pending l->frame */
 
-	list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-		if ((l->transport == transport)
-		    && (l->client_pid == pid)) {
-			__delete_lock (pl_inode, l);
-			__destroy_lock (l);
-		}
-	}
+int32_t
+pl_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 
-	list_for_each_entry_safe (l, tmp, &pl_inode->int_list, list) {
-		if ((l->transport == transport)
-		    && (l->client_pid == pid)) {
-			__delete_lock (pl_inode, l);
-			__destroy_lock (l);
-		}
-	}
+}
 
-	return;
+int32_t
+pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             const char *name, dict_t *xdata)
+{
+        int32_t                 op_errno        = EINVAL;
+        int                     op_ret          = -1;
+        int32_t                 bcount          = 0;
+        int32_t                 gcount          = 0;
+        char                    key[PATH_MAX]   = {0, };
+        char                    *lk_summary     = NULL;
+        pl_inode_t              *pl_inode       = NULL;
+        dict_t                  *dict           = NULL;
+        clrlk_args              args            = {0,};
+        char                    *brickname      = NULL;
+
+        if (!name)
+                goto usual;
+
+        if (strncmp (name, GF_XATTR_CLRLK_CMD, strlen (GF_XATTR_CLRLK_CMD)))
+                goto usual;
+
+        if (clrlk_parse_args (name, &args)) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        pl_inode = pl_inode_get (this, loc->inode);
+        if (!pl_inode) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        switch (args.type) {
+                case CLRLK_INODE:
+                case CLRLK_ENTRY:
+                        op_ret = clrlk_clear_lks_in_all_domains (this, pl_inode,
+                                                                 &args, &bcount,
+                                                                 &gcount,
+                                                                 &op_errno);
+                        if (op_ret)
+                                goto out;
+                        break;
+                case CLRLK_POSIX:
+                        op_ret = clrlk_clear_posixlk (this, pl_inode, &args,
+                                                      &bcount, &gcount,
+                                                      &op_errno);
+                        if (op_ret)
+                                goto out;
+                        break;
+                case CLRLK_TYPE_MAX:
+                        op_errno = EINVAL;
+                        goto out;
+        }
+
+        op_ret = fetch_pathinfo (this, loc->inode, &op_errno, &brickname);
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Couldn't get brickname");
+        } else {
+                op_ret = format_brickname(brickname);
+                if (op_ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Couldn't format brickname");
+                        GF_FREE(brickname);
+                        brickname = NULL;
+                }
+        }
+
+        if (!gcount && !bcount) {
+                if (gf_asprintf (&lk_summary, "No locks cleared.") == -1) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto out;
+                }
+        } else if (gf_asprintf (&lk_summary, "%s: %s blocked locks=%d "
+                                "granted locks=%d",
+                                (brickname == NULL)? this->name : brickname,
+                                (args.type == CLRLK_INODE)? "inode":
+                                (args.type == CLRLK_ENTRY)? "entry":
+                                (args.type == CLRLK_POSIX)? "posix": " ",
+                                bcount, gcount) == -1) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        strncpy (key, name, strlen (name));
+        if (dict_set_dynstr (dict, key, lk_summary)) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = 0;
+out:
+        GF_FREE(brickname);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+
+        GF_FREE (args.opts);
+        if (op_ret && lk_summary)
+                GF_FREE (lk_summary);
+        if (dict)
+                dict_unref (dict);
+        return 0;
+
+usual:
+        STACK_WIND (frame, pl_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
 }
 
+static int
+format_brickname(char *brickname)
+{
+       int   ret             = -1;
+       char *hostname        = NULL;
+       char *volume          = NULL;
+       char *saveptr         = NULL;
 
-int
-pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int32_t op_ret, int32_t op_errno)
+       if (!brickname)
+               goto out;
+
+       strtok_r(brickname, ":", &saveptr);
+       hostname = gf_strdup(strtok_r(NULL, ":", &saveptr));
+       if (hostname == NULL)
+               goto out;
+       volume = gf_strdup(strtok_r(NULL, ".", &saveptr));
+       if (volume == NULL)
+               goto out;
+
+       sprintf(brickname, "%s:%s", hostname, volume);
+
+       ret = 0;
+out:
+       GF_FREE(hostname);
+       GF_FREE(volume);
+       return ret;
+}
+
+static int
+fetch_pathinfo (xlator_t *this, inode_t *inode, int32_t *op_errno,
+                char **brickname)
 {
-	STACK_UNWIND (frame, op_ret, op_errno);
+        int                    ret       = -1;
+        loc_t                  loc       = {0, };
+        dict_t                *dict      = NULL;
+
+        if (!brickname)
+                goto out;
+
+        if (!op_errno)
+                goto out;
+
+        uuid_copy (loc.gfid, inode->gfid);
+        loc.inode = inode_ref (inode);
+
+        ret = syncop_getxattr (FIRST_CHILD(this), &loc, &dict,
+                               GF_XATTR_PATHINFO_KEY);
+        if (ret < 0) {
+                *op_errno = errno;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, brickname);
+        if (ret)
+                goto out;
+
+        *brickname = gf_strdup(*brickname);
+        if (*brickname == NULL) {
+                ret = -1;
+                goto out;
+        }
 
-	return 0;
+        ret = 0;
+out:
+        if (dict != NULL) {
+                dict_unref (dict);
+        }
+        loc_wipe(&loc);
+
+        return ret;
 }
 
 
 int
-pl_flush (call_frame_t *frame, xlator_t *this,
-	  fd_t *fd)
+pl_lockinfo_get_brickname (xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+        int                    ret       = -1;
+        posix_locks_private_t *priv      = NULL;
+        char                  *brickname = NULL;
+        char                  *end       = NULL;
+        char                  *tmp       = NULL;
+
+        priv = this->private;
+
+        ret = fetch_pathinfo (this, inode, op_errno, &brickname);
+        if (ret)
+                goto out;
+
+        end = strrchr (brickname, ':');
+        if (!end) {
+                GF_FREE(brickname);
+                ret = -1;
+                goto out;
+        }
+
+        tmp = brickname;
+        brickname = gf_strndup (brickname, (end - brickname));
+        if (brickname == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        priv->brickname = brickname;
+        ret = 0;
+out:
+        GF_FREE(tmp);
+        return ret;
+}
+
+char *
+pl_lockinfo_key (xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+        posix_locks_private_t *priv = NULL;
+        char                  *key  = NULL;
+        int                    ret = 0;
+
+        priv = this->private;
+
+        if (priv->brickname == NULL) {
+                ret = pl_lockinfo_get_brickname (this, inode, op_errno);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cannot get brickname");
+                        goto out;
+                }
+        }
+
+        key = priv->brickname;
+out:
+        return key;
+}
+
+int32_t
+pl_fgetxattr_handle_lockinfo (xlator_t *this, fd_t *fd,
+                              dict_t *dict, int32_t *op_errno)
+{
+        pl_inode_t    *pl_inode = NULL;
+        char          *key      = NULL, *buf = NULL;
+        int32_t        op_ret   = 0;
+        unsigned long  fdnum    = 0, len = 0;
+        dict_t        *tmp      = NULL;
+
+        pl_inode = pl_inode_get (this, fd->inode);
+
+        if (!pl_inode) {
+                gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
+                *op_errno = EBADFD;
+                op_ret = -1;
+                goto out;
+        }
+
+        if (!pl_locks_by_fd (pl_inode, fd)) {
+                op_ret = 0;
+                goto out;
+        }
+
+        fdnum = fd_to_fdnum (fd);
+
+        key = pl_lockinfo_key (this, fd->inode, op_errno);
+        if (key == NULL) {
+                op_ret = -1;
+                goto out;
+        }
+
+        tmp = dict_new ();
+        if (tmp == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_set_uint64 (tmp, key, fdnum);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
+                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+                        fdnum, fd, uuid_utoa (fd->inode->gfid),
+                        strerror (*op_errno));
+                goto out;
+        }
+
+        len = dict_serialized_length (tmp);
+        if (len < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict_serialized_length failed (%s) while handling "
+                        "lockinfo for fd (ptr:%p inode-gfid:%s)",
+                        strerror (*op_errno), fd, uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+        buf = GF_CALLOC (1, len, gf_common_mt_char);
+        if (buf == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_serialize (tmp, buf);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict_serialize failed (%s) while handling lockinfo "
+                        "for fd (ptr: %p inode-gfid:%s)", strerror (*op_errno),
+                        fd, uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+        op_ret = dict_set_dynptr (dict, GF_XATTR_LOCKINFO_KEY, buf, len);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
+                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+                        fdnum, fd, uuid_utoa (fd->inode->gfid),
+                        strerror (*op_errno));
+                goto out;
+        }
+
+        buf = NULL;
+out:
+        if (tmp != NULL) {
+                dict_unref (tmp);
+        }
+
+        if (buf != NULL) {
+                GF_FREE (buf);
+        }
+
+        return op_ret;
+}
+
+
+int32_t
+pl_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              const char *name, dict_t *xdata)
 {
-	posix_locks_private_t *priv = NULL;
-	pl_inode_t            *pl_inode = NULL;
+        int32_t  op_ret = 0, op_errno = 0;
+        dict_t  *dict   = NULL;
+
+        if (!name) {
+                goto usual;
+        }
+
+        if (strcmp (name, GF_XATTR_LOCKINFO_KEY) == 0) {
+                dict = dict_new ();
+                if (dict == NULL) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = pl_fgetxattr_handle_lockinfo (this, fd, dict,
+                                                       &op_errno);
+                if (op_ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "getting lockinfo on fd (ptr:%p inode-gfid:%s) "
+                                "failed (%s)", fd, uuid_utoa (fd->inode->gfid),
+                                strerror (op_errno));
+                }
+
+                goto unwind;
+        } else {
+                goto usual;
+        }
 
-	priv = this->private;
+unwind:
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, NULL);
+        if (dict != NULL) {
+                dict_unref (dict);
+        }
 
-	pl_inode = pl_inode_get (this, fd->inode);
-	if (!pl_inode) {
-		gf_log (this->name, GF_LOG_ERROR, "returning EBADFD");
-		STACK_UNWIND (frame, -1, EBADFD);
-		return 0;
-	}
+        return 0;
 
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
-		__delete_locks_of_owner (pl_inode, frame->root->trans,
-					 frame->root->pid);
-	}
-	pthread_mutex_unlock (&pl_inode->mutex);
+usual:
+        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+        return 0;
+}
 
-	grant_blocked_locks (this, pl_inode, GF_LOCK_POSIX);
-	grant_blocked_locks (this, pl_inode, GF_LOCK_INTERNAL);
+int32_t
+pl_migrate_locks (call_frame_t *frame, fd_t *newfd, uint64_t oldfd_num,
+                  int32_t *op_errno)
+{
+        pl_inode_t   *pl_inode  = NULL;
+        uint64_t      newfd_num = 0;
+        posix_lock_t *l         = NULL;
+        int32_t       op_ret    = 0;
+
+        newfd_num = fd_to_fdnum (newfd);
+
+        pl_inode = pl_inode_get (frame->this, newfd->inode);
+        if (pl_inode == NULL) {
+                op_ret = -1;
+                *op_errno = EBADFD;
+                goto out;
+        }
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (l, &pl_inode->ext_list, list) {
+                        if (l->fd_num == oldfd_num) {
+                                l->fd_num = newfd_num;
+                                l->client = frame->root->client;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        op_ret = 0;
+out:
+        return op_ret;
+}
 
-	do_blocked_rw (pl_inode);
+int32_t
+pl_fsetxattr_handle_lockinfo (call_frame_t *frame, fd_t *fd, char *lockinfo_buf,
+                              int len, int32_t *op_errno)
+{
+        int32_t   op_ret    = -1;
+        dict_t   *lockinfo  = NULL;
+        uint64_t  oldfd_num = 0;
+        char     *key       = NULL;
+
+        lockinfo = dict_new ();
+        if (lockinfo == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_unserialize (lockinfo_buf, len, &lockinfo);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                goto out;
+        }
+
+        key = pl_lockinfo_key (frame->this, fd->inode, op_errno);
+        if (key == NULL) {
+                op_ret = -1;
+                goto out;
+        }
+
+        op_ret = dict_get_uint64 (lockinfo, key, &oldfd_num);
+
+        if (oldfd_num == 0) {
+                op_ret = 0;
+                goto out;
+        }
+
+        op_ret = pl_migrate_locks (frame, fd, oldfd_num, op_errno);
+        if (op_ret < 0) {
+                gf_log (frame->this->name, GF_LOG_WARNING,
+                        "migration of locks from oldfd (ptr:%p) to newfd "
+                        "(ptr:%p) (inode-gfid:%s)", (void *)oldfd_num, fd,
+                        uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
 
-	STACK_WIND (frame, pl_flush_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->flush, fd);
-	return 0;
+out:
+        dict_unref (lockinfo);
+
+        return op_ret;
+}
+
+int32_t
+pl_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
+{
+        int32_t  op_ret       = 0, op_errno = 0;
+        void    *lockinfo_buf = NULL;
+        int      len          = 0;
+
+        op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                       &lockinfo_buf, &len);
+        if (lockinfo_buf == NULL) {
+                goto usual;
+        }
+
+        op_ret = pl_fsetxattr_handle_lockinfo (frame, fd, lockinfo_buf, len,
+                                               &op_errno);
+        if (op_ret < 0) {
+                goto unwind;
+        }
+
+usual:
+        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+        return 0;
+
+unwind:
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
+        return 0;
+}
+
+int32_t
+pl_opendir_cbk (call_frame_t *frame,
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                fd_t *fd, dict_t *xdata)
+{
+        pl_fdctx_t *fdctx = NULL;
+
+        if (op_ret < 0)
+                goto unwind;
+
+        fdctx = pl_check_n_create_fdctx (this, fd);
+        if (!fdctx) {
+                op_errno = ENOMEM;
+                op_ret   = -1;
+                goto unwind;
+        }
+
+unwind:
+        STACK_UNWIND_STRICT (opendir,
+                             frame,
+                             op_ret,
+                             op_errno,
+                             fd, xdata);
+        return 0;
+}
+
+int32_t
+pl_opendir (call_frame_t *frame, xlator_t *this,
+             loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+        STACK_WIND (frame,
+                    pl_opendir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+        return 0;
+
+}
+
+int
+pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+
+int
+pl_flush (call_frame_t *frame, xlator_t *this,
+          fd_t *fd, dict_t *xdata)
+{
+        pl_inode_t *pl_inode = NULL;
+
+        pl_inode = pl_inode_get (this, fd->inode);
+
+        if (!pl_inode) {
+                gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
+                STACK_UNWIND_STRICT (flush, frame, -1, EBADFD, NULL);
+                return 0;
+        }
+
+        pl_trace_flush (this, frame, fd);
+
+        if (frame->root->lk_owner.len == 0) {
+                /* Handle special case when protocol/server sets lk-owner to zero.
+                 * This usually happens due to a client disconnection. Hence, free
+                 * all locks opened with this fd.
+                 */
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Releasing all locks with fd %p", fd);
+                delete_locks_of_fd (this, pl_inode, fd);
+                goto wind;
+
+        }
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __delete_locks_of_owner (pl_inode, frame->root->client,
+                                         &frame->root->lk_owner);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        grant_blocked_locks (this, pl_inode);
+
+        do_blocked_rw (pl_inode);
+
+wind:
+        STACK_WIND (frame, pl_flush_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->flush, fd, xdata);
+        return 0;
 }
 
 
 int
 pl_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	     int32_t op_ret, int32_t op_errno, fd_t *fd)
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
+        pl_fdctx_t *fdctx = NULL;
+
+        if (op_ret < 0)
+                goto unwind;
+
+        fdctx = pl_check_n_create_fdctx (this, fd);
+        if (!fdctx) {
+                op_errno = ENOMEM;
+                op_ret   = -1;
+                goto unwind;
+        }
 
-	return 0;
+unwind:
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+
+        return 0;
 }
 
 
 int
-pl_open (call_frame_t *frame, xlator_t *this,
-	 loc_t *loc, int32_t flags, fd_t *fd)
+pl_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
 {
-	/* why isn't O_TRUNC being handled ? */
-	STACK_WIND (frame, pl_open_cbk, 
-		    FIRST_CHILD(this), FIRST_CHILD(this)->fops->open, 
-		    loc, flags & ~O_TRUNC, fd);
+        STACK_WIND (frame, pl_open_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
 
-	return 0;
+        return 0;
 }
 
 
 int
 pl_create_cbk (call_frame_t *frame, void *cookie,
-	       xlator_t *this, int32_t op_ret, int32_t op_errno,
-	       fd_t *fd, inode_t *inode, struct stat *buf)
+               xlator_t *this, int32_t op_ret, int32_t op_errno,
+               fd_t *fd, inode_t *inode, struct iatt *buf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
+        pl_fdctx_t *fdctx = NULL;
+
+        if (op_ret < 0)
+                goto unwind;
+
+        fdctx = pl_check_n_create_fdctx (this, fd);
+        if (!fdctx) {
+                op_errno = ENOMEM;
+                op_ret   = -1;
+                goto unwind;
+        }
+
+unwind:
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
 
-	return 0;
+        return 0;
 }
 
 
 int
 pl_create (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int32_t flags, mode_t mode, fd_t *fd)
+           loc_t *loc, int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+           dict_t *xdata)
 {
-	STACK_WIND (frame, pl_create_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create, 
-		    loc, flags, mode, fd);
-	return 0;
+        STACK_WIND (frame, pl_create_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
 }
 
 
 int
 pl_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int32_t op_ret, int32_t op_errno,
-	      struct iovec *vector, int32_t count, struct stat *stbuf)
+              int32_t op_ret, int32_t op_errno,
+              struct iovec *vector, int32_t count, struct iatt *stbuf,
+              struct iobref *iobref, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf);
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
+                             vector, count, stbuf, iobref, xdata);
 
-	return 0;
+        return 0;
 }
 
 int
-pl_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-	       int32_t op_ret, int32_t op_errno, struct stat *stbuf)
+pl_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, stbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 
-	return 0;
+        return 0;
 }
 
 
 void
 do_blocked_rw (pl_inode_t *pl_inode)
 {
-	struct list_head  wind_list;
-	pl_rw_req_t      *rw = NULL;
-	pl_rw_req_t      *tmp = NULL;
-
-	INIT_LIST_HEAD (&wind_list);
-
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
-		list_for_each_entry_safe (rw, tmp, &pl_inode->rw_list, list) {
-			if (__rw_allowable (pl_inode, &rw->region,
-					    rw->stub->fop)) {
-				list_del_init (&rw->list);
-				list_add_tail (&rw->list, &wind_list);
-			}
-		}
-	}
-	pthread_mutex_unlock (&pl_inode->mutex);
-
-	list_for_each_entry_safe (rw, tmp, &wind_list, list) {
-		list_del_init (&rw->list);
-		call_resume (rw->stub);
-		free (rw);
-	}
-
-	return;
+        struct list_head  wind_list;
+        pl_rw_req_t      *rw = NULL;
+        pl_rw_req_t      *tmp = NULL;
+
+        INIT_LIST_HEAD (&wind_list);
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (rw, tmp, &pl_inode->rw_list, list) {
+                        if (__rw_allowable (pl_inode, &rw->region,
+                                            rw->stub->fop)) {
+                                list_del_init (&rw->list);
+                                list_add_tail (&rw->list, &wind_list);
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (rw, tmp, &wind_list, list) {
+                list_del_init (&rw->list);
+                call_resume (rw->stub);
+                GF_FREE (rw);
+        }
+
+        return;
 }
 
 
 static int
 __rw_allowable (pl_inode_t *pl_inode, posix_lock_t *region,
-		glusterfs_fop_t op)
+                glusterfs_fop_t op)
 {
-	posix_lock_t *l = NULL;
-	int           ret = 1;
-
-	list_for_each_entry (l, &pl_inode->ext_list, list) {
-		if (locks_overlap (l, region) && !same_owner (l, region)) {
-			if ((op == GF_FOP_READ) && (l->fl_type != F_WRLCK))
-				continue;
-			ret = 0;
-			break;
-		}
-	}
-
-	return ret;
+        posix_lock_t *l = NULL;
+        int           ret = 1;
+
+        list_for_each_entry (l, &pl_inode->ext_list, list) {
+                if (locks_overlap (l, region) && !same_owner (l, region)) {
+                        if ((op == GF_FOP_READ) && (l->fl_type != F_WRLCK))
+                                continue;
+                        ret = 0;
+                        break;
+                }
+        }
+
+        return ret;
 }
 
 
 int
-pl_readv_cont (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, size_t size, off_t offset)
+pl_readv_cont (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, uint32_t flags, dict_t *xdata)
 {
-	STACK_WIND (frame, pl_readv_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset);
+        STACK_WIND (frame, pl_readv_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
+                    fd, size, offset, flags, xdata);
 
-	return 0;
+        return 0;
 }
 
 
 int
 pl_readv (call_frame_t *frame, xlator_t *this,
-	  fd_t *fd, size_t size, off_t offset)
-{
-	posix_locks_private_t *priv = NULL;
-	pl_inode_t            *pl_inode = NULL;
-	pl_rw_req_t           *rw = NULL;
-	posix_lock_t           region = {.list = {0, }, };
-	int                    op_ret = 0;
-	int                    op_errno = 0;
-	char                   allowable = 0;
-
-
-	priv = this->private;
-
-	pl_inode = pl_inode_get (this, fd->inode);
-
-	if (priv->mandatory && pl_inode->mandatory) {
-		region.fl_start   = offset;
-		region.fl_end     = offset + size - 1;
-		region.transport  = frame->root->trans;
-		region.client_pid = frame->root->pid;
-    
-		pthread_mutex_lock (&pl_inode->mutex);
-		{
-			allowable = __rw_allowable (pl_inode, &region,
-						    GF_FOP_READ);
-			if (allowable)
-				goto unlock;
-
-			if (fd->flags & O_NONBLOCK) {
-				gf_log (this->name, GF_LOG_DEBUG,
-					"returning EWOULDBLOCK");
-				op_errno = EWOULDBLOCK;
-				op_ret = -1;
-				goto unlock;
-			}
-
-			rw = CALLOC (1, sizeof (*rw));
-			if (!rw) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"out of memory :(");
-				op_errno = ENOMEM;
-				op_ret = -1;
-				goto unlock;
-			}
-
-			rw->stub = fop_readv_stub (frame, pl_readv_cont,
-						   fd, size, offset);
-			if (!rw->stub) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"out of memory :(");
-				op_errno = ENOMEM;
-				op_ret = -1;
-				free (rw);
-				goto unlock;
-			}
-
-			rw->region = region;
-
-			list_add_tail (&rw->list, &pl_inode->rw_list);
-		}
-	unlock:
-		pthread_mutex_unlock (&pl_inode->mutex);
-
-		goto unwind;
-	}
-
-
-	STACK_WIND (frame, pl_readv_cbk, 
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset);
-	return 0;
-
-unwind:
-	if (op_ret == -1)
-		STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL);
-
-	return 0;
+          fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        posix_locks_private_t *priv = NULL;
+        pl_inode_t            *pl_inode = NULL;
+        pl_rw_req_t           *rw = NULL;
+        posix_lock_t           region = {.list = {0, }, };
+        int                    op_ret = 0;
+        int                    op_errno = 0;
+        char                   wind_needed = 1;
+
+
+        priv = this->private;
+        pl_inode = pl_inode_get (this, fd->inode);
+
+        if (priv->mandatory && pl_inode->mandatory) {
+                region.fl_start   = offset;
+                region.fl_end     = offset + size - 1;
+                region.client     = frame->root->client;
+                region.fd_num     = fd_to_fdnum(fd);
+                region.client_pid = frame->root->pid;
+                region.owner      = frame->root->lk_owner;
+
+                pthread_mutex_lock (&pl_inode->mutex);
+                {
+                        wind_needed = __rw_allowable (pl_inode, &region,
+                                                      GF_FOP_READ);
+                        if (wind_needed) {
+                                goto unlock;
+                        }
+
+                        if (fd->flags & O_NONBLOCK) {
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "returning EAGAIN as fd is O_NONBLOCK");
+                                op_errno = EAGAIN;
+                                op_ret = -1;
+                                goto unlock;
+                        }
+
+                        rw = GF_CALLOC (1, sizeof (*rw),
+                                        gf_locks_mt_pl_rw_req_t);
+                        if (!rw) {
+                                op_errno = ENOMEM;
+                                op_ret = -1;
+                                goto unlock;
+                        }
+
+                        rw->stub = fop_readv_stub (frame, pl_readv_cont,
+                                                   fd, size, offset, flags,
+                                                   xdata);
+                        if (!rw->stub) {
+                                op_errno = ENOMEM;
+                                op_ret = -1;
+                                GF_FREE (rw);
+                                goto unlock;
+                        }
+
+                        rw->region = region;
+
+                        list_add_tail (&rw->list, &pl_inode->rw_list);
+                }
+        unlock:
+                pthread_mutex_unlock (&pl_inode->mutex);
+        }
+
+
+        if (wind_needed) {
+                STACK_WIND (frame, pl_readv_cbk,
+                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
+                            fd, size, offset, flags, xdata);
+        }
+
+        if (op_ret == -1)
+                STACK_UNWIND_STRICT (readv, frame, -1, op_errno,
+                                     NULL, 0, NULL, NULL, NULL);
+
+        return 0;
 }
 
 
 int
 pl_writev_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		struct iovec *vector, int count, off_t offset)
+                struct iovec *vector, int count, off_t offset,
+                uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-	STACK_WIND (frame, pl_writev_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-		    fd, vector, count, offset);
+        STACK_WIND (frame, pl_writev_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
+                    fd, vector, count, offset, flags, iobref, xdata);
 
-	return 0;
+        return 0;
 }
 
 
 int
 pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	   struct iovec *vector, int32_t count, off_t offset)
-{
-	posix_locks_private_t *priv = NULL;
-	pl_inode_t            *pl_inode = NULL;
-	pl_rw_req_t           *rw = NULL;
-	posix_lock_t           region = {.list = {0, }, };
-	int                    op_ret = 0;
-	int                    op_errno = 0;
-	char                   allowable = 0;
-
-
-	priv = this->private;
-
-	pl_inode = pl_inode_get (this, fd->inode);
-	if (priv->mandatory && pl_inode->mandatory) {
-		region.fl_start   = offset;
-		region.fl_end     = offset + iov_length (vector, count) - 1;
-		region.transport  = frame->root->trans;
-		region.client_pid = frame->root->pid;
-    
-		pthread_mutex_lock (&pl_inode->mutex);
-		{
-			allowable = __rw_allowable (pl_inode, &region,
-						    GF_FOP_WRITE);
-			if (allowable)
-				goto unlock;
-
-			if (fd->flags & O_NONBLOCK) {
-				gf_log (this->name, GF_LOG_DEBUG,
-					"returning EWOULDBLOCK");
-				op_errno = EWOULDBLOCK;
-				op_ret = -1;
-				goto unlock;
-			}
-
-			rw = CALLOC (1, sizeof (*rw));
-			if (!rw) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"out of memory :(");
-				op_errno = ENOMEM;
-				op_ret = -1;
-				goto unlock;
-			}
-
-			rw->stub = fop_writev_stub (frame, pl_writev_cont,
-						    fd, vector, count, offset);
-			if (!rw->stub) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"out of memory :(");
-				op_errno = ENOMEM;
-				op_ret = -1;
-				free (rw);
-				goto unlock;
-			}
-
-			rw->region = region;
-
-			list_add_tail (&rw->list, &pl_inode->rw_list);
-		}
-	unlock:
-		pthread_mutex_unlock (&pl_inode->mutex);
-
-		goto unwind;
-	}
-
-
-	STACK_WIND (frame, pl_writev_cbk, 
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-		    fd, vector, count, offset);
-	return 0;
+           struct iovec *vector, int32_t count, off_t offset,
+           uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        posix_locks_private_t *priv = NULL;
+        pl_inode_t            *pl_inode = NULL;
+        pl_rw_req_t           *rw = NULL;
+        posix_lock_t           region = {.list = {0, }, };
+        int                    op_ret = 0;
+        int                    op_errno = 0;
+        char                   wind_needed = 1;
+
+        priv = this->private;
+        pl_inode = pl_inode_get (this, fd->inode);
+
+        if (priv->mandatory && pl_inode->mandatory) {
+                region.fl_start   = offset;
+                region.fl_end     = offset + iov_length (vector, count) - 1;
+                region.client     = frame->root->client;
+                region.fd_num     = fd_to_fdnum(fd);
+                region.client_pid = frame->root->pid;
+                region.owner      = frame->root->lk_owner;
+
+                pthread_mutex_lock (&pl_inode->mutex);
+                {
+                        wind_needed = __rw_allowable (pl_inode, &region,
+                                                      GF_FOP_WRITE);
+                        if (wind_needed)
+                                goto unlock;
+
+                        if (fd->flags & O_NONBLOCK) {
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "returning EAGAIN because fd is "
+                                        "O_NONBLOCK");
+                                op_errno = EAGAIN;
+                                op_ret = -1;
+                                goto unlock;
+                        }
+
+                        rw = GF_CALLOC (1, sizeof (*rw),
+                                        gf_locks_mt_pl_rw_req_t);
+                        if (!rw) {
+                                op_errno = ENOMEM;
+                                op_ret = -1;
+                                goto unlock;
+                        }
+
+                        rw->stub = fop_writev_stub (frame, pl_writev_cont,
+                                                    fd, vector, count, offset,
+                                                    flags, iobref, xdata);
+                        if (!rw->stub) {
+                                op_errno = ENOMEM;
+                                op_ret = -1;
+                                GF_FREE (rw);
+                                goto unlock;
+                        }
+
+                        rw->region = region;
+
+                        list_add_tail (&rw->list, &pl_inode->rw_list);
+                }
+        unlock:
+                pthread_mutex_unlock (&pl_inode->mutex);
+        }
+
+
+        if (wind_needed)
+                STACK_WIND (frame, pl_writev_cbk,
+                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
+                            fd, vector, count, offset, flags, iobref, xdata);
+
+        if (op_ret == -1)
+                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL,
+                                     NULL);
+
+        return 0;
+}
 
-unwind:
-	if (op_ret == -1)
-		STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL);
+static int
+__fd_has_locks (pl_inode_t *pl_inode, fd_t *fd)
+{
+        int           found = 0;
+        posix_lock_t *l     = NULL;
+
+        list_for_each_entry (l, &pl_inode->ext_list, list) {
+                if ((l->fd_num == fd_to_fdnum(fd))) {
+                        found = 1;
+                        break;
+                }
+        }
 
-	return 0;
+        return found;
 }
 
+static posix_lock_t *
+lock_dup (posix_lock_t *lock)
+{
+        posix_lock_t *new_lock = NULL;
+
+        new_lock = new_posix_lock (&lock->user_flock, lock->client,
+                                   lock->client_pid, &lock->owner,
+                                   (fd_t *)lock->fd_num);
+        return new_lock;
+}
+
+static int
+__dup_locks_to_fdctx (pl_inode_t *pl_inode, fd_t *fd,
+                      pl_fdctx_t *fdctx)
+{
+        posix_lock_t *l        = NULL;
+        posix_lock_t *duplock = NULL;
+        int ret = 0;
+
+        list_for_each_entry (l, &pl_inode->ext_list, list) {
+                if ((l->fd_num == fd_to_fdnum(fd))) {
+                        duplock = lock_dup (l);
+                        if (!duplock) {
+                                ret = -1;
+                                break;
+                        }
+
+                        list_add_tail (&duplock->list, &fdctx->locks_list);
+                }
+        }
+
+        return ret;
+}
+
+static int
+__copy_locks_to_fdctx (pl_inode_t *pl_inode, fd_t *fd,
+                      pl_fdctx_t *fdctx)
+{
+        int ret = 0;
+
+        ret = __dup_locks_to_fdctx (pl_inode, fd, fdctx);
+        if (ret)
+                goto out;
+
+out:
+        return ret;
+
+}
+
+static void
+pl_mark_eol_lock (posix_lock_t *lock)
+{
+        lock->user_flock.l_type = GF_LK_EOL;
+        return;
+}
+
+static posix_lock_t *
+__get_next_fdctx_lock (pl_fdctx_t *fdctx)
+{
+        posix_lock_t *lock = NULL;
+
+        GF_ASSERT (fdctx);
+
+        if (list_empty (&fdctx->locks_list)) {
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "fdctx lock list empty");
+                goto out;
+        }
+
+        lock = list_entry (fdctx->locks_list.next, typeof (*lock),
+                           list);
+
+        GF_ASSERT (lock);
+
+        list_del_init (&lock->list);
+
+out:
+        return lock;
+}
+
+static int
+__set_next_lock_fd (pl_fdctx_t *fdctx, posix_lock_t *reqlock)
+{
+        posix_lock_t *lock  = NULL;
+        int           ret   = 0;
+
+        GF_ASSERT (fdctx);
+
+        lock = __get_next_fdctx_lock (fdctx);
+        if (!lock) {
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "marking EOL in reqlock");
+                pl_mark_eol_lock (reqlock);
+                goto out;
+        }
+
+        reqlock->user_flock  = lock->user_flock;
+        reqlock->fl_start    = lock->fl_start;
+        reqlock->fl_type     = lock->fl_type;
+        reqlock->fl_end      = lock->fl_end;
+        reqlock->owner       = lock->owner;
+
+out:
+        if (lock)
+                __destroy_lock (lock);
+
+        return ret;
+}
+
+static int
+pl_getlk_fd (xlator_t *this, pl_inode_t *pl_inode,
+             fd_t *fd, posix_lock_t *reqlock)
+{
+        uint64_t    tmp   = 0;
+        pl_fdctx_t *fdctx = NULL;
+        int         ret   = 0;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                if (!__fd_has_locks (pl_inode, fd)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "fd=%p has no active locks", fd);
+                        ret = 0;
+                        goto unlock;
+                }
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "There are active locks on fd");
+
+                ret = fd_ctx_get (fd, this, &tmp);
+                fdctx = (pl_fdctx_t *)(long) tmp;
+
+                if (list_empty (&fdctx->locks_list)) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "no fdctx -> copying all locks on fd");
+
+                        ret = __copy_locks_to_fdctx (pl_inode, fd, fdctx);
+                        if (ret) {
+                                goto unlock;
+                        }
+
+                        ret = __set_next_lock_fd (fdctx, reqlock);
+
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "fdctx present -> returning the next lock");
+                        ret = __set_next_lock_fd (fdctx, reqlock);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "could not get next lock of fd");
+                                goto unlock;
+                        }
+                }
+        }
+
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
+        return ret;
+
+}
 
 int
 pl_lk (call_frame_t *frame, xlator_t *this,
-       fd_t *fd, int32_t cmd, struct flock *flock)
-{
-	transport_t           *transport = NULL;
-	pid_t                  client_pid = 0;
-	posix_locks_private_t *priv = NULL;
-	pl_inode_t            *pl_inode = NULL;
-	int                    op_ret = 0;
-	int                    op_errno = 0;
-	int                    can_block = 0;
-	posix_lock_t          *reqlock = NULL;
-	posix_lock_t          *conf = NULL;
-	int                    ret = 0;
-
-	transport  = frame->root->trans;
-	client_pid = frame->root->pid;
-	priv       = this->private;
-
-	pl_inode = pl_inode_get (this, fd->inode);
-	if (!pl_inode) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		op_ret = -1;
-		op_errno = ENOMEM;
-		goto unwind;
-	}
-
-	reqlock = new_posix_lock (flock, transport, client_pid);
-	if (!reqlock) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory :(");
-		op_ret = -1;
-		op_errno = ENOMEM;
-		goto unwind;
-	}
-
-	switch (cmd) {
+       fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+        pl_inode_t   *pl_inode   = NULL;
+        int           op_ret     = 0;
+        int           op_errno   = 0;
+        int           can_block  = 0;
+        posix_lock_t *reqlock    = NULL;
+        posix_lock_t *conf       = NULL;
+        int           ret        = 0;
+
+        if ((flock->l_start < 0) || (flock->l_len < 0)) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto unwind;
+        }
+
+        pl_inode = pl_inode_get (this, fd->inode);
+        if (!pl_inode) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        reqlock = new_posix_lock (flock, frame->root->client, frame->root->pid,
+                                  &frame->root->lk_owner, fd);
+
+        if (!reqlock) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        pl_trace_in (this, frame, fd, NULL, cmd, flock, NULL);
+
+        switch (cmd) {
+
+        case F_RESLK_LCKW:
+                can_block = 1;
+
+                /* fall through */
+        case F_RESLK_LCK:
+                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
+                reqlock->frame = frame;
+                reqlock->this = this;
+
+                ret = pl_reserve_setlk (this, pl_inode, reqlock,
+                                        can_block);
+                if (ret < 0) {
+                        if (can_block)
+                                goto out;
+
+                        op_ret = -1;
+                        op_errno = -ret;
+                        __destroy_lock (reqlock);
+                        goto unwind;
+                }
+                /* Finally a getlk and return the call */
+                conf = pl_getlk (pl_inode, reqlock);
+                if (conf)
+                        posix_lock_to_flock (conf, flock);
+                break;
+
+        case F_RESLK_UNLCK:
+                reqlock->frame = frame;
+                reqlock->this = this;
+                ret = pl_reserve_unlock (this, pl_inode, reqlock);
+                if (ret < 0) {
+                        op_ret = -1;
+                        op_errno = -ret;
+                }
+                __destroy_lock (reqlock);
+                goto unwind;
+
+                break;
+
+        case F_GETLK_FD:
+                reqlock->frame = frame;
+                reqlock->this = this;
+                ret = pl_verify_reservelk (this, pl_inode, reqlock, can_block);
+                GF_ASSERT (ret >= 0);
+
+                ret = pl_getlk_fd (this, pl_inode, fd, reqlock);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "getting locks on fd failed");
+                        op_ret = -1;
+                        op_errno = ENOLCK;
+                        goto unwind;
+                }
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Replying with a lock on fd for healing");
+
+                posix_lock_to_flock (reqlock, flock);
+                __destroy_lock (reqlock);
+
+                break;
 
 #if F_GETLK != F_GETLK64
-	case F_GETLK64:
+        case F_GETLK64:
 #endif
-	case F_GETLK:
-		conf = pl_getlk (pl_inode, reqlock, GF_LOCK_POSIX);
-		posix_lock_to_flock (conf, flock);
-		__destroy_lock (reqlock);
+        case F_GETLK:
+                conf = pl_getlk (pl_inode, reqlock);
+                posix_lock_to_flock (conf, flock);
+                __destroy_lock (reqlock);
 
-		break;
+                break;
 
 #if F_SETLKW != F_SETLKW64
-	case F_SETLKW64:
+        case F_SETLKW64:
 #endif
-	case F_SETLKW:
-		can_block = 1;
-		reqlock->frame = frame;
-		reqlock->this  = this;
-		reqlock->fd    = fd;
+        case F_SETLKW:
+                can_block = 1;
+                reqlock->frame  = frame;
+                reqlock->this   = this;
 
-		/* fall through */
+                /* fall through */
 
 #if F_SETLK != F_SETLK64
-	case F_SETLK64:
+        case F_SETLK64:
 #endif
-	case F_SETLK:
-		memcpy (&reqlock->user_flock, flock, sizeof (struct flock));
-		ret = pl_setlk (this, pl_inode, reqlock,
-				can_block, GF_LOCK_POSIX);
-
-		if (ret == -1) {
-			if (can_block)
-				goto out;
-
-			gf_log (this->name, GF_LOG_DEBUG, "returning EAGAIN");
-			op_ret = -1;
-			op_errno = EAGAIN;
-			__destroy_lock (reqlock);
-		}
-	}
+        case F_SETLK:
+                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
+                ret = pl_verify_reservelk (this, pl_inode, reqlock, can_block);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "Lock blocked due to conflicting reserve lock");
+                        goto out;
+                }
+                ret = pl_setlk (this, pl_inode, reqlock,
+                                can_block);
+
+                if (ret == -1) {
+                        if ((can_block) && (F_UNLCK != flock->l_type)) {
+                                pl_trace_block (this, frame, fd, NULL, cmd, flock, NULL);
+                                goto out;
+                        }
+                        gf_log (this->name, GF_LOG_DEBUG, "returning EAGAIN");
+                        op_ret = -1;
+                        op_errno = EAGAIN;
+                        __destroy_lock (reqlock);
+
+                } else if ((0 == ret) && (F_UNLCK == flock->l_type)) {
+                        /* For NLM's last "unlock on fd" detection */
+                        if (pl_locks_by_fd (pl_inode, fd))
+                                flock->l_type = F_RDLCK;
+                        else
+                                flock->l_type = F_UNLCK;
+                }
+        }
 
 unwind:
-	STACK_UNWIND (frame, op_ret, op_errno, flock);
+        pl_trace_out (this, frame, fd, NULL, cmd, flock, op_ret, op_errno, NULL);
+        pl_update_refkeeper (this, fd->inode);
+
+
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock, xdata);
 out:
-	return 0;
+        return 0;
 }
 
 
 /* TODO: this function just logs, no action required?? */
 int
 pl_forget (xlator_t *this,
-	   inode_t *inode)
+           inode_t *inode)
+{
+        pl_inode_t   *pl_inode = NULL;
+
+        posix_lock_t *ext_tmp = NULL;
+        posix_lock_t *ext_l   = NULL;
+        struct list_head posixlks_released;
+
+        pl_inode_lock_t *ino_tmp = NULL;
+        pl_inode_lock_t *ino_l   = NULL;
+        struct list_head inodelks_released;
+
+        pl_rw_req_t *rw_tmp = NULL;
+        pl_rw_req_t *rw_req = NULL;
+
+        pl_entry_lock_t *entry_tmp = NULL;
+        pl_entry_lock_t *entry_l   = NULL;
+        struct list_head entrylks_released;
+
+        pl_dom_list_t *dom = NULL;
+        pl_dom_list_t *dom_tmp = NULL;
+
+        INIT_LIST_HEAD (&posixlks_released);
+        INIT_LIST_HEAD (&inodelks_released);
+        INIT_LIST_HEAD (&entrylks_released);
+
+        pl_inode = pl_inode_get (this, inode);
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+
+                if (!list_empty (&pl_inode->rw_list)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Pending R/W requests found, releasing.");
+
+                        list_for_each_entry_safe (rw_req, rw_tmp, &pl_inode->rw_list,
+                                                  list) {
+
+                                list_del (&rw_req->list);
+                                GF_FREE (rw_req);
+                        }
+                }
+
+                if (!list_empty (&pl_inode->ext_list)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Pending fcntl locks found, releasing.");
+
+                        list_for_each_entry_safe (ext_l, ext_tmp, &pl_inode->ext_list,
+                                                  list) {
+
+                                __delete_lock (pl_inode, ext_l);
+                                if (ext_l->blocked) {
+                                        list_add_tail (&ext_l->list, &posixlks_released);
+                                        continue;
+                                }
+                                __destroy_lock (ext_l);
+                        }
+                }
+
+
+                list_for_each_entry_safe (dom, dom_tmp, &pl_inode->dom_list, inode_list) {
+
+                        if (!list_empty (&dom->inodelk_list)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Pending inode locks found, releasing.");
+
+                                list_for_each_entry_safe (ino_l, ino_tmp, &dom->inodelk_list, list) {
+                                        __delete_inode_lock (ino_l);
+                                        __pl_inodelk_unref (ino_l);
+                                }
+
+                                list_splice_init (&dom->blocked_inodelks, &inodelks_released);
+
+
+                        }
+                        if (!list_empty (&dom->entrylk_list)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Pending entry locks found, releasing.");
+
+                                list_for_each_entry_safe (entry_l, entry_tmp, &dom->entrylk_list, domain_list) {
+                                        list_del_init (&entry_l->domain_list);
+
+                                        GF_FREE ((char *)entry_l->basename);
+                                        GF_FREE (entry_l->connection_id);
+                                        GF_FREE (entry_l);
+                                }
+
+                                list_splice_init (&dom->blocked_entrylks, &entrylks_released);
+                        }
+
+                        list_del (&dom->inode_list);
+                        gf_log ("posix-locks", GF_LOG_TRACE,
+                                " Cleaning up domain: %s", dom->domain);
+                        GF_FREE ((char *)(dom->domain));
+                        GF_FREE (dom);
+                }
+
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (ext_l, ext_tmp, &posixlks_released, list) {
+
+                STACK_UNWIND_STRICT (lk, ext_l->frame, -1, 0,
+                                     &ext_l->user_flock, NULL);
+                __destroy_lock (ext_l);
+        }
+
+        list_for_each_entry_safe (ino_l, ino_tmp, &inodelks_released, blocked_locks) {
+
+                STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0, NULL);
+                __pl_inodelk_unref (ino_l);
+        }
+
+        list_for_each_entry_safe (entry_l, entry_tmp, &entrylks_released, blocked_locks) {
+
+                STACK_UNWIND_STRICT (entrylk, entry_l->frame, -1, 0, NULL);
+                GF_FREE ((char *)entry_l->basename);
+                GF_FREE (entry_l->connection_id);
+                GF_FREE (entry_l);
+
+        }
+
+        GF_FREE (pl_inode);
+
+        return 0;
+}
+
+int
+pl_release (xlator_t *this, fd_t *fd)
+{
+        pl_inode_t *pl_inode     = NULL;
+        uint64_t    tmp_pl_inode = 0;
+        int         ret          = -1;
+        uint64_t    tmp          = 0;
+        pl_fdctx_t *fdctx        = NULL;
+
+        if (fd == NULL) {
+                goto out;
+        }
+
+        ret = inode_ctx_get (fd->inode, this, &tmp_pl_inode);
+        if (ret != 0)
+                goto out;
+
+        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+
+        pl_trace_release (this, fd);
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "Releasing all locks with fd %p", fd);
+
+        delete_locks_of_fd (this, pl_inode, fd);
+        pl_update_refkeeper (this, fd->inode);
+
+        ret = fd_ctx_del (fd, this, &tmp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Could not get fdctx");
+                goto out;
+        }
+
+        fdctx = (pl_fdctx_t *)(long)tmp;
+
+        GF_FREE (fdctx);
+out:
+        return ret;
+}
+
+int
+pl_releasedir (xlator_t *this, fd_t *fd)
+{
+        int         ret          = -1;
+        uint64_t    tmp          = 0;
+        pl_fdctx_t *fdctx        = NULL;
+
+        if (fd == NULL) {
+                goto out;
+        }
+
+        ret = fd_ctx_del (fd, this, &tmp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Could not get fdctx");
+                goto out;
+        }
+
+        fdctx = (pl_fdctx_t *)(long)tmp;
+
+        GF_FREE (fdctx);
+out:
+        return ret;
+}
+
+int32_t
+__get_posixlk_count (xlator_t *this, pl_inode_t *pl_inode)
+{
+        posix_lock_t *lock   = NULL;
+        int32_t       count  = 0;
+
+        list_for_each_entry (lock, &pl_inode->ext_list, list) {
+
+                count++;
+        }
+
+        return count;
+}
+
+int32_t
+get_posixlk_count (xlator_t *this, inode_t *inode)
 {
-	pl_inode_t   *pl_inode = NULL;
+        pl_inode_t   *pl_inode = NULL;
+        uint64_t      tmp_pl_inode = 0;
+        int           ret      = 0;
+        int32_t       count    = 0;
 
-	pl_inode = pl_inode_get (this, inode);
+        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
+        if (ret != 0) {
+                goto out;
+        }
 
-	if (!list_empty (&pl_inode->rw_list)) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"pending R/W requests found!");
-	}
+        pl_inode = (pl_inode_t *)(long) tmp_pl_inode;
 
-	if (!list_empty (&pl_inode->ext_list)) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Pending fcntl locks found!");
-	}
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                count =__get_posixlk_count (this, pl_inode);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
 
-	if (!list_empty (&pl_inode->int_list)) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Pending internal locks found!");
-	}
+out:
+        return count;
+}
+
+void
+pl_parent_entrylk_xattr_fill (xlator_t *this, inode_t *parent,
+                              char *basename, dict_t *dict)
+{
+        uint32_t         entrylk = 0;
+        int             ret     = -1;
+
+        if (!parent || !basename || !strlen (basename))
+                goto out;
+        entrylk = check_entrylk_on_basename (this, parent, basename);
+out:
+        ret = dict_set_uint32 (dict, GLUSTERFS_PARENT_ENTRYLK, entrylk);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        " dict_set failed on key %s", GLUSTERFS_PARENT_ENTRYLK);
+        }
+}
+
+void
+pl_entrylk_xattr_fill (xlator_t *this, inode_t *inode,
+                       dict_t *dict)
+{
+        int32_t     count = 0;
+        int         ret   = -1;
+
+        count = get_entrylk_count (this, inode);
+        ret = dict_set_int32 (dict, GLUSTERFS_ENTRYLK_COUNT, count);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        " dict_set failed on key %s", GLUSTERFS_ENTRYLK_COUNT);
+        }
+
+}
+
+void
+pl_inodelk_xattr_fill (xlator_t *this, inode_t *inode, dict_t *dict,
+                       gf_boolean_t per_dom)
+{
+        int32_t     count = 0;
+        int         ret   = -1;
+        char        *domname = NULL;
+
+
+        if (per_dom){
+                ret = dict_get_str (dict, GLUSTERFS_INODELK_DOM_COUNT,
+                                    &domname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "value for key %s",GLUSTERFS_INODELK_DOM_COUNT);
+                        goto out;
+                }
+        }
+
+        count = get_inodelk_count (this, inode, domname);
+
+        ret = dict_set_int32 (dict, GLUSTERFS_INODELK_COUNT, count);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to set count for "
+                        "key %s", GLUSTERFS_INODELK_COUNT);
+        }
+
+out:
+        return;
+}
+
+void
+pl_posixlk_xattr_fill (xlator_t *this, inode_t *inode,
+                       dict_t *dict)
+{
+        int32_t     count = 0;
+        int         ret   = -1;
+
+        count = get_posixlk_count (this, inode);
+        ret = dict_set_int32 (dict, GLUSTERFS_POSIXLK_COUNT, count);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        " dict_set failed on key %s", GLUSTERFS_POSIXLK_COUNT);
+        }
+
+}
+
+int32_t
+pl_lookup_cbk (call_frame_t *frame,
+                    void *cookie,
+                    xlator_t *this,
+                    int32_t op_ret,
+                    int32_t op_errno,
+                    inode_t *inode,
+                    struct iatt *buf,
+                    dict_t *xdata,
+                    struct iatt *postparent)
+{
+        pl_local_t *local = NULL;
 
-	if (!list_empty (&pl_inode->dir_list)) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Pending entry locks found!");
-	}
+        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
 
-	FREE (pl_inode);
+        if (op_ret)
+                goto out;
 
-	return 0;
+        local = frame->local;
+
+        if (local->parent_entrylk_req)
+                pl_parent_entrylk_xattr_fill (this, local->loc.parent,
+                                              (char*)local->loc.name, xdata);
+        if (local->entrylk_count_req)
+                pl_entrylk_xattr_fill (this, inode, xdata);
+        if (local->inodelk_count_req)
+                pl_inodelk_xattr_fill (this, inode, xdata, _gf_false);
+        if (local->inodelk_dom_count_req)
+                pl_inodelk_xattr_fill (this, inode, xdata, _gf_true);
+        if (local->posixlk_count_req)
+                pl_posixlk_xattr_fill (this, inode, xdata);
+
+
+out:
+        local = frame->local;
+        frame->local = NULL;
+
+        if (local != NULL) {
+                loc_wipe (&local->loc);
+                mem_put (local);
+        }
+
+        STACK_UNWIND_STRICT (
+                     lookup,
+                     frame,
+                     op_ret,
+                     op_errno,
+                     inode,
+                     buf,
+                     xdata,
+                     postparent);
+        return 0;
 }
 
+int32_t
+pl_lookup (call_frame_t *frame,
+           xlator_t *this,
+           loc_t *loc,
+           dict_t *xdata)
+{
+        pl_local_t *local  = NULL;
+        int         ret    = -1;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (this, out);
+        VALIDATE_OR_GOTO (loc, out);
+
+        local = mem_get0 (this->local_pool);
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+        if (xdata) {
+                if (dict_get (xdata, GLUSTERFS_ENTRYLK_COUNT))
+                        local->entrylk_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_INODELK_COUNT))
+                        local->inodelk_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_INODELK_DOM_COUNT))
+                        local->inodelk_dom_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_POSIXLK_COUNT))
+                        local->posixlk_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_PARENT_ENTRYLK))
+                        local->parent_entrylk_req = 1;
+        }
+
+        frame->local = local;
+        loc_copy (&local->loc, loc);
+
+        STACK_WIND (frame,
+                    pl_lookup_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    loc, xdata);
+        ret = 0;
+out:
+        if (ret == -1)
+                STACK_UNWIND_STRICT (lookup, frame, -1, 0, NULL,
+                                     NULL, NULL, NULL);
 
+        return 0;
+}
 int
-init (xlator_t *this)
+pl_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-	posix_locks_private_t *priv = NULL;
-	xlator_list_t         *trav = NULL;
-	data_t                *mandatory = NULL;
+        pl_local_t *local  = NULL;
+        gf_dirent_t *entry = NULL;
+
+        local = frame->local;
+
+        if (op_ret <= 0)
+                goto unwind;
+
+        list_for_each_entry (entry, &entries->list, list) {
+                if (local->entrylk_count_req)
+                        pl_entrylk_xattr_fill (this, entry->inode, entry->dict);
+                if (local->inodelk_count_req)
+                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
+                                               _gf_false);
+                if (local->inodelk_dom_count_req)
+                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
+                                               _gf_true);
+                if (local->posixlk_count_req)
+                        pl_posixlk_xattr_fill (this, entry->inode, entry->dict);
+        }
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+
+        if (local)
+                mem_put (local);
 
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"FATAL: posix-locks should have exactly one child");
-		return -1;
-	}
+        return 0;
+}
 
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
+int
+pl_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict)
+{
+        pl_local_t *local  = NULL;
 
-	trav = this->children;
-	while (trav->xlator->children)
-		trav = trav->xlator->children;
+        local = mem_get0 (this->local_pool);
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
 
-	if (strncmp ("storage/", trav->xlator->type, 8)) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"'posix-locks' not loaded over storage translator");
-		return -1;
-	}
+        if (dict) {
+                if (dict_get (dict, GLUSTERFS_ENTRYLK_COUNT))
+                        local->entrylk_count_req = 1;
+                if (dict_get (dict, GLUSTERFS_INODELK_COUNT))
+                        local->inodelk_count_req = 1;
+                if (dict_get (dict, GLUSTERFS_INODELK_DOM_COUNT))
+                        local->inodelk_dom_count_req = 1;
+                if (dict_get (dict, GLUSTERFS_POSIXLK_COUNT))
+                        local->posixlk_count_req = 1;
+        }
 
-	priv = CALLOC (1, sizeof (*priv));
+        frame->local = local;
 
-	mandatory = dict_get (this->options, "mandatory-locks");
-	if (mandatory) {
-		if (gf_string2boolean (mandatory->data,
-				       &priv->mandatory) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"'mandatory-locks' takes only boolean "
-				"options");
-			return -1;
-		}
-	}
+        STACK_WIND (frame, pl_readdirp_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
+                    fd, size, offset, dict);
 
-	this->private = priv;
-	return 0;
+        return 0;
+out:
+        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+}
+
+
+void
+pl_dump_lock (char *str, int size, struct gf_flock *flock,
+              gf_lkowner_t *owner, void *trans, char *conn_id,
+              time_t *granted_time, time_t *blkd_time, gf_boolean_t active)
+{
+        char  *type_str    = NULL;
+        char   granted[32] = {0,};
+        char   blocked[32] = {0,};
+
+        switch (flock->l_type) {
+        case F_RDLCK:
+                type_str = "READ";
+                break;
+        case F_WRLCK:
+                type_str = "WRITE";
+                break;
+        case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+        default:
+                type_str = "UNKNOWN";
+                break;
+        }
+
+        if (active) {
+                if (blkd_time && *blkd_time == 0) {
+                        snprintf (str, size, RANGE_GRNTD_FMT,
+                                  type_str, flock->l_whence,
+                                  (unsigned long long) flock->l_start,
+                                  (unsigned long long) flock->l_len,
+                                  (unsigned long long) flock->l_pid,
+                                  lkowner_utoa (owner), trans, conn_id,
+                                  ctime_r (granted_time, granted));
+                } else {
+                        snprintf (str, size, RANGE_BLKD_GRNTD_FMT,
+                                  type_str, flock->l_whence,
+                                  (unsigned long long) flock->l_start,
+                                  (unsigned long long) flock->l_len,
+                                  (unsigned long long) flock->l_pid,
+                                  lkowner_utoa (owner), trans, conn_id,
+                                  ctime_r (blkd_time, blocked),
+                                  ctime_r (granted_time, granted));
+                }
+        }
+        else {
+                snprintf (str, size, RANGE_BLKD_FMT,
+                          type_str, flock->l_whence,
+                          (unsigned long long) flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid,
+                          lkowner_utoa (owner), trans, conn_id,
+                          ctime_r (blkd_time, blocked));
+        }
+
+}
+
+void
+__dump_entrylks (pl_inode_t *pl_inode)
+{
+        pl_dom_list_t   *dom  = NULL;
+        pl_entry_lock_t *lock = NULL;
+        char             blocked[32] = {0,};
+        char             granted[32] = {0,};
+        int              count = 0;
+        char             key[GF_DUMP_MAX_BUF_LEN] = {0,};
+
+        char tmp[256];
+
+        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+
+                count = 0;
+
+                gf_proc_dump_build_key(key,
+                                       "lock-dump.domain",
+                                       "domain");
+                gf_proc_dump_write(key, "%s", dom->domain);
+
+                list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
+
+                        gf_proc_dump_build_key(key,
+                                               "xlator.feature.locks.lock-dump.domain.entrylk",
+                                               "entrylk[%d](ACTIVE)", count );
+                        if (lock->blkd_time.tv_sec == 0 && lock->blkd_time.tv_usec == 0) {
+                                snprintf (tmp, 256, ENTRY_GRNTD_FMT,
+                                          lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
+                                          "ENTRYLK_WRLCK", lock->basename,
+                                          (unsigned long long) lock->client_pid,
+                                          lkowner_utoa (&lock->owner), lock->trans,
+                                          lock->connection_id,
+                                          ctime_r (&lock->granted_time.tv_sec, granted));
+                        } else {
+                                snprintf (tmp, 256, ENTRY_BLKD_GRNTD_FMT,
+                                          lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
+                                          "ENTRYLK_WRLCK", lock->basename,
+                                          (unsigned long long) lock->client_pid,
+                                          lkowner_utoa (&lock->owner), lock->trans,
+                                          lock->connection_id,
+                                          ctime_r (&lock->blkd_time.tv_sec, blocked),
+                                          ctime_r (&lock->granted_time.tv_sec, granted));
+                        }
+
+                        gf_proc_dump_write(key, tmp);
+
+                        count++;
+                }
+
+                list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
+
+                        gf_proc_dump_build_key(key,
+                                               "xlator.feature.locks.lock-dump.domain.entrylk",
+                                               "entrylk[%d](BLOCKED)", count );
+                        snprintf (tmp, 256, ENTRY_BLKD_FMT,
+                                  lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
+                                  "ENTRYLK_WRLCK", lock->basename,
+                                  (unsigned long long) lock->client_pid,
+                                  lkowner_utoa (&lock->owner), lock->trans,
+                                  lock->connection_id,
+                                  ctime_r (&lock->blkd_time.tv_sec, blocked));
+
+                        gf_proc_dump_write(key, tmp);
+
+                        count++;
+                }
+
+        }
+
+}
+
+void
+dump_entrylks (pl_inode_t *pl_inode)
+{
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __dump_entrylks (pl_inode);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+}
+
+void
+__dump_inodelks (pl_inode_t *pl_inode)
+{
+        pl_dom_list_t   *dom  = NULL;
+        pl_inode_lock_t *lock = NULL;
+        int             count = 0;
+        char            key[GF_DUMP_MAX_BUF_LEN];
+
+        char tmp[256];
+
+        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+
+                count = 0;
+
+                gf_proc_dump_build_key(key,
+                                       "lock-dump.domain",
+                                       "domain");
+                gf_proc_dump_write(key, "%s", dom->domain);
+
+                list_for_each_entry (lock, &dom->inodelk_list, list) {
+
+                        gf_proc_dump_build_key(key,
+                                               "inodelk",
+                                               "inodelk[%d](ACTIVE)",count );
+
+                        SET_FLOCK_PID (&lock->user_flock, lock);
+                        pl_dump_lock (tmp, 256, &lock->user_flock,
+                                      &lock->owner,
+                                      lock->client, lock->connection_id,
+                                      &lock->granted_time.tv_sec,
+                                      &lock->blkd_time.tv_sec,
+                                      _gf_true);
+                        gf_proc_dump_write(key, tmp);
+
+                        count++;
+                }
+
+                list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+
+                        gf_proc_dump_build_key(key,
+                                               "inodelk",
+                                               "inodelk[%d](BLOCKED)",count );
+                        SET_FLOCK_PID (&lock->user_flock, lock);
+                        pl_dump_lock (tmp, 256, &lock->user_flock,
+                                      &lock->owner,
+                                      lock->client, lock->connection_id,
+                                      0, &lock->blkd_time.tv_sec,
+                                      _gf_false);
+                        gf_proc_dump_write(key, tmp);
+
+                        count++;
+                }
+
+        }
+
+}
+
+void
+dump_inodelks (pl_inode_t *pl_inode)
+{
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __dump_inodelks (pl_inode);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+}
+
+void
+__dump_posixlks (pl_inode_t *pl_inode)
+{
+        posix_lock_t    *lock = NULL;
+        int             count = 0;
+        char            key[GF_DUMP_MAX_BUF_LEN];
+
+        char tmp[256];
+
+      list_for_each_entry (lock, &pl_inode->ext_list, list) {
+
+              SET_FLOCK_PID (&lock->user_flock, lock);
+              gf_proc_dump_build_key(key,
+                                     "posixlk",
+                                     "posixlk[%d](%s)",
+                                     count,
+                                     lock->blocked ? "BLOCKED" : "ACTIVE");
+              pl_dump_lock (tmp, 256, &lock->user_flock,
+                            &lock->owner, lock->client, NULL,
+                            &lock->granted_time.tv_sec, &lock->blkd_time.tv_sec,
+                            (lock->blocked)? _gf_false: _gf_true);
+              gf_proc_dump_write(key, tmp);
+
+              count++;
+        }
+}
+
+void
+dump_posixlks (pl_inode_t *pl_inode)
+{
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __dump_posixlks (pl_inode);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+}
+
+int32_t
+pl_dump_inode_priv (xlator_t *this, inode_t *inode)
+{
+
+        int             ret = -1;
+        uint64_t        tmp_pl_inode = 0;
+        pl_inode_t      *pl_inode = NULL;
+        char            *pathname = NULL;
+        gf_boolean_t    section_added = _gf_false;
+
+        int count      = 0;
+
+        if (!inode) {
+                errno = EINVAL;
+                goto out;
+        }
+
+        ret = TRY_LOCK (&inode->lock);
+        if (ret)
+                goto out;
+        {
+                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
+                if (ret)
+                        goto unlock;
+        }
+unlock:
+        UNLOCK (&inode->lock);
+        if (ret)
+                goto out;
+
+        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+        if (!pl_inode) {
+                ret = -1;
+                goto out;
+        }
+
+        gf_proc_dump_add_section("xlator.features.locks.%s.inode", this->name);
+        section_added = _gf_true;
+
+        /*We are safe to call __inode_path since we have the
+         * inode->table->lock */
+        __inode_path (inode, NULL, &pathname);
+        if (pathname)
+                gf_proc_dump_write ("path", "%s", pathname);
+
+        gf_proc_dump_write("mandatory", "%d", pl_inode->mandatory);
+
+        ret = pthread_mutex_trylock (&pl_inode->mutex);
+        if (ret)
+                goto out;
+        {
+                count = __get_entrylk_count (this, pl_inode);
+                if (count) {
+                        gf_proc_dump_write("entrylk-count", "%d", count);
+                        __dump_entrylks (pl_inode);
+                }
+
+                count = __get_inodelk_count (this, pl_inode, NULL);
+                if (count) {
+                        gf_proc_dump_write("inodelk-count", "%d", count);
+                        __dump_inodelks (pl_inode);
+                }
+
+                count = __get_posixlk_count (this, pl_inode);
+                if (count) {
+                        gf_proc_dump_write("posixlk-count", "%d", count);
+                        __dump_posixlks (pl_inode);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+out:
+        GF_FREE (pathname);
+
+        if (ret && inode) {
+                if (!section_added)
+                        gf_proc_dump_add_section ("xlator.features.locks.%s."
+                                                  "inode", this->name);
+                gf_proc_dump_write ("Unable to print lock state", "(Lock "
+                                    "acquisition failure) %s",
+                                    uuid_utoa (inode->gfid));
+        }
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_locks_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                                "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+
+pl_ctx_t*
+pl_ctx_get (client_t *client, xlator_t *xlator)
+{
+        void *tmp = NULL;
+        pl_ctx_t *ctx = NULL;
+
+        client_ctx_get (client, xlator, &tmp);
+
+        ctx = tmp;
+
+        if (ctx != NULL)
+                goto out;
+
+        ctx = GF_CALLOC (1, sizeof (pl_ctx_t), gf_locks_mt_posix_lock_t);
+
+        if (ctx == NULL)
+                goto out;
+
+        ctx->ltable = pl_lock_table_new();
+
+        if (ctx->ltable == NULL) {
+                GF_FREE (ctx);
+                ctx = NULL;
+                goto out;
+        }
+
+        LOCK_INIT (&ctx->ltable_lock);
+
+        if (client_ctx_set (client, xlator, ctx) != 0) {
+                LOCK_DESTROY (&ctx->ltable_lock);
+                GF_FREE (ctx->ltable);
+                GF_FREE (ctx);
+                ctx = NULL;
+        }
+out:
+        return ctx;
+}
+
+static void
+ltable_delete_locks (struct _lock_table *ltable)
+{
+        struct _locker *locker = NULL;
+        struct _locker *tmp    = NULL;
+
+        list_for_each_entry_safe (locker, tmp, &ltable->inodelk_lockers, lockers) {
+                if (locker->fd)
+                        pl_del_locker (ltable, locker->volume, &locker->loc,
+                                       locker->fd, &locker->owner,
+                                       GF_FOP_INODELK);
+                GF_FREE (locker->volume);
+                GF_FREE (locker);
+        }
+
+        list_for_each_entry_safe (locker, tmp, &ltable->entrylk_lockers, lockers) {
+                if (locker->fd)
+                        pl_del_locker (ltable, locker->volume, &locker->loc,
+                                       locker->fd, &locker->owner,
+                                       GF_FOP_ENTRYLK);
+                GF_FREE (locker->volume);
+                GF_FREE (locker);
+        }
+        GF_FREE (ltable);
+}
+
+
+static int32_t
+destroy_cbk (xlator_t *this, client_t *client)
+{
+        void     *tmp       = NULL;
+        pl_ctx_t *locks_ctx = NULL;
+
+        client_ctx_del (client, this, &tmp);
+
+        if (tmp == NULL)
+                return 0
+;
+        locks_ctx = tmp;
+        if (locks_ctx->ltable)
+                ltable_delete_locks (locks_ctx->ltable);
+
+        LOCK_DESTROY (&locks_ctx->ltable_lock);
+        GF_FREE (locks_ctx);
+
+        return 0;
+}
+
+
+static int32_t
+disconnect_cbk (xlator_t *this, client_t *client)
+{
+        int32_t              ret       = 0;
+        pl_ctx_t            *locks_ctx = NULL;
+        struct _lock_table  *ltable    = NULL;
+
+        locks_ctx = pl_ctx_get (client, this);
+        if (locks_ctx == NULL) {
+                gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+                goto out;
+        }
+
+        LOCK (&locks_ctx->ltable_lock);
+        {
+                if (locks_ctx->ltable) {
+                        ltable = locks_ctx->ltable;
+                        locks_ctx->ltable = pl_lock_table_new ();
+                }
+        }
+        UNLOCK (&locks_ctx->ltable_lock);
+
+        if (ltable)
+                ltable_delete_locks (ltable);
+
+out:
+        return ret;
+}
+
+
+int
+init (xlator_t *this)
+{
+        posix_locks_private_t *priv = NULL;
+        xlator_list_t         *trav = NULL;
+        data_t                *mandatory = NULL;
+        data_t                *trace = NULL;
+        int                   ret = -1;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "FATAL: posix-locks should have exactly one child");
+                goto out;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Volume is dangling. Please check the volume file.");
+        }
+
+        trav = this->children;
+        while (trav->xlator->children)
+                trav = trav->xlator->children;
+
+        if (strncmp ("storage/", trav->xlator->type, 8)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "'locks' translator is not loaded over a storage "
+                        "translator");
+                goto out;
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv),
+                          gf_locks_mt_posix_locks_private_t);
+
+        mandatory = dict_get (this->options, "mandatory-locks");
+        if (mandatory)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "mandatory locks not supported in this minor release.");
+
+        trace = dict_get (this->options, "trace");
+        if (trace) {
+                if (gf_string2boolean (trace->data,
+                                       &priv->trace) == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "'trace' takes on only boolean values.");
+                        goto out;
+                }
+        }
+
+        this->local_pool = mem_pool_new (pl_local_t, 32);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
+        this->private = priv;
+        ret = 0;
+
+out:
+        if (ret) {
+                GF_FREE (priv);
+        }
+        return ret;
 }
 
 
 int
 fini (xlator_t *this)
 {
-	posix_locks_private_t *priv = NULL;
+        posix_locks_private_t *priv = NULL;
 
-	priv = this->private;
-	free (priv);
+        priv = this->private;
+        if (!priv)
+                return 0;
+        this->private = NULL;
+        GF_FREE (priv->brickname);
+        GF_FREE (priv);
 
-	return 0;
+        return 0;
 }
 
 
 int
-pl_inodelk (call_frame_t *frame, xlator_t *this, 
-	    loc_t *loc, int32_t cmd, struct flock *flock);
+pl_inodelk (call_frame_t *frame, xlator_t *this,
+            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
+            dict_t *xdata);
 
 int
-pl_finodelk (call_frame_t *frame, xlator_t *this, 
-	     fd_t *fd, int32_t cmd, struct flock *flock);
+pl_finodelk (call_frame_t *frame, xlator_t *this,
+             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+             dict_t *xdata);
 
 int
-pl_entrylk (call_frame_t *frame, xlator_t *this, 
-	    loc_t *loc, const char *basename, 
-	    entrylk_cmd cmd, entrylk_type type);
+pl_entrylk (call_frame_t *frame, xlator_t *this,
+            const char *volume, loc_t *loc, const char *basename,
+            entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 int
-pl_fentrylk (call_frame_t *frame, xlator_t *this, 
-	     fd_t *fd, const char *basename, 
-	     entrylk_cmd cmd, entrylk_type type);
+pl_fentrylk (call_frame_t *frame, xlator_t *this,
+             const char *volume, fd_t *fd, const char *basename,
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 struct xlator_fops fops = {
-	.create      = pl_create,
-	.truncate    = pl_truncate,
-	.ftruncate   = pl_ftruncate,
-	.open        = pl_open,
-	.readv       = pl_readv,
-	.writev      = pl_writev,
-	.lk          = pl_lk,
-	.inodelk     = pl_inodelk,
-	.finodelk    = pl_finodelk,
-	.entrylk     = pl_entrylk,
-	.fentrylk    = pl_fentrylk,
-	.flush       = pl_flush,
+        .lookup      = pl_lookup,
+        .create      = pl_create,
+        .truncate    = pl_truncate,
+        .ftruncate   = pl_ftruncate,
+        .open        = pl_open,
+        .readv       = pl_readv,
+        .writev      = pl_writev,
+        .lk          = pl_lk,
+        .inodelk     = pl_inodelk,
+        .finodelk    = pl_finodelk,
+        .entrylk     = pl_entrylk,
+        .fentrylk    = pl_fentrylk,
+        .flush       = pl_flush,
+        .opendir     = pl_opendir,
+        .readdirp    = pl_readdirp,
+        .getxattr    = pl_getxattr,
+        .fgetxattr   = pl_fgetxattr,
+        .fsetxattr   = pl_fsetxattr,
 };
 
-
-struct xlator_mops mops = {
+struct xlator_dumpops dumpops = {
+        .inodectx    = pl_dump_inode_priv,
 };
 
-
 struct xlator_cbks cbks = {
-	.forget      = pl_forget,
+        .forget            = pl_forget,
+        .release           = pl_release,
+        .releasedir        = pl_releasedir,
+        .client_destroy    = destroy_cbk,
+        .client_disconnect = disconnect_cbk,
 };
 
 
 struct volume_options options[] = {
-	{ .key  = { "mandatory-locks", "mandatory" }, 
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key = {NULL} },
+        { .key  = { "mandatory-locks", "mandatory" },
+          .type = GF_OPTION_TYPE_BOOL
+        },
+        { .key  = { "trace" },
+          .type = GF_OPTION_TYPE_BOOL
+        },
+        { .key = {NULL} },
 };
diff --git a/xlators/features/locks/src/reservelk.c b/xlators/features/locks/src/reservelk.c
new file mode 100644
index 000000000..11abd26d8
--- /dev/null
+++ b/xlators/features/locks/src/reservelk.c
@@ -0,0 +1,443 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "compat.h"
+#include "xlator.h"
+#include "inode.h"
+#include "logging.h"
+#include "common-utils.h"
+#include "list.h"
+
+#include "locks.h"
+#include "common.h"
+
+void
+__delete_reserve_lock (posix_lock_t *lock)
+{
+        list_del (&lock->list);
+}
+
+void
+__destroy_reserve_lock (posix_lock_t *lock)
+{
+        GF_FREE (lock);
+}
+
+/* Return true if the two reservelks have exactly same lock boundaries */
+int
+reservelks_equal (posix_lock_t *l1, posix_lock_t *l2)
+{
+        if ((l1->fl_start == l2->fl_start) &&
+            (l1->fl_end == l2->fl_end))
+                return 1;
+
+        return 0;
+}
+
+/* Determine if lock is grantable or not */
+static posix_lock_t *
+__reservelk_grantable (pl_inode_t *pl_inode, posix_lock_t *lock)
+{
+        xlator_t     *this     = NULL;
+        posix_lock_t *l        = NULL;
+        posix_lock_t *ret_lock = NULL;
+
+        this = THIS;
+
+        if (list_empty (&pl_inode->reservelk_list)) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "No reservelks in list");
+                goto out;
+        }
+        list_for_each_entry (l, &pl_inode->reservelk_list, list){
+                if (reservelks_equal (lock, l)) {
+                        ret_lock = l;
+                        break;
+                }
+        }
+out:
+        return ret_lock;
+}
+
+static inline int
+__same_owner_reservelk (posix_lock_t *l1, posix_lock_t *l2)
+{
+        return (is_same_lkowner (&l1->owner, &l2->owner));
+
+}
+
+static posix_lock_t *
+__matching_reservelk (pl_inode_t *pl_inode, posix_lock_t *lock)
+{
+        posix_lock_t *l = NULL;
+
+        if (list_empty (&pl_inode->reservelk_list)) {
+            gf_log ("posix-locks", GF_LOG_TRACE,
+                    "reservelk list empty");
+            return NULL;
+        }
+
+        list_for_each_entry (l, &pl_inode->reservelk_list, list) {
+                if (reservelks_equal (l, lock)) {
+                        gf_log ("posix-locks", GF_LOG_TRACE,
+                                "equal reservelk found");
+                        break;
+                }
+        }
+
+        return l;
+}
+
+static int
+__reservelk_conflict (xlator_t *this, pl_inode_t *pl_inode,
+                      posix_lock_t *lock)
+{
+        posix_lock_t *conf = NULL;
+        int ret = 0;
+
+        conf = __matching_reservelk (pl_inode, lock);
+        if (conf) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Matching reservelk found");
+                if (__same_owner_reservelk (lock, conf)) {
+                        list_del_init (&conf->list);
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "Removing the matching reservelk for setlk to progress");
+                        GF_FREE (conf);
+                        ret = 0;
+                } else {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "Conflicting reservelk found");
+                        ret = 1;
+                }
+
+        }
+        return ret;
+
+}
+
+int
+pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
+                     posix_lock_t *lock, int can_block)
+{
+        int ret = 0;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                if (__reservelk_conflict (this, pl_inode, lock)) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "Found conflicting reservelk. Blocking until reservelk is unlocked.");
+                        lock->blocked = can_block;
+                        list_add_tail (&lock->list, &pl_inode->blocked_calls);
+                        ret = -1;
+                        goto unlock;
+                }
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "no conflicting reservelk found. Call continuing");
+                ret = 0;
+
+        }
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        return ret;
+
+}
+
+
+/* Determines if lock can be granted and adds the lock. If the lock
+ * is blocking, adds it to the blocked_reservelks.
+ */
+static int
+__lock_reservelk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                  int can_block)
+{
+        posix_lock_t *conf = NULL;
+        int ret = -EINVAL;
+
+        conf = __reservelk_grantable (pl_inode, lock);
+        if (conf){
+                ret = -EAGAIN;
+                if (can_block == 0)
+                        goto out;
+
+                list_add_tail (&lock->list, &pl_inode->blocked_reservelks);
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
+                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                        lock->client_pid,
+                        lkowner_utoa (&lock->owner),
+                        lock->user_flock.l_start,
+                        lock->user_flock.l_len);
+
+
+                goto out;
+        }
+
+        list_add (&lock->list, &pl_inode->reservelk_list);
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static posix_lock_t *
+find_matching_reservelk (posix_lock_t *lock, pl_inode_t *pl_inode)
+{
+        posix_lock_t *l = NULL;
+        list_for_each_entry (l, &pl_inode->reservelk_list, list) {
+                if (reservelks_equal (l, lock))
+                        return l;
+        }
+        return NULL;
+}
+
+/* Set F_UNLCK removes a lock which has the exact same lock boundaries
+ * as the UNLCK lock specifies. If such a lock is not found, returns invalid
+ */
+static posix_lock_t *
+__reserve_unlock_lock (xlator_t *this, posix_lock_t *lock, pl_inode_t *pl_inode)
+{
+
+        posix_lock_t *conf = NULL;
+
+        conf = find_matching_reservelk (lock, pl_inode);
+        if (!conf) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        " Matching lock not found for unlock");
+                goto out;
+        }
+        __delete_reserve_lock (conf);
+        gf_log (this->name, GF_LOG_DEBUG,
+                " Matching lock found for unlock");
+
+out:
+        return conf;
+
+
+}
+
+static void
+__grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode,
+                               struct list_head *granted)
+{
+        int              bl_ret = 0;
+        posix_lock_t *bl = NULL;
+        posix_lock_t *tmp = NULL;
+
+        struct list_head blocked_list;
+
+        INIT_LIST_HEAD (&blocked_list);
+        list_splice_init (&pl_inode->blocked_reservelks, &blocked_list);
+
+        list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
+
+                list_del_init (&bl->list);
+
+                bl_ret = __lock_reservelk (this, pl_inode, bl, 1);
+
+                if (bl_ret == 0) {
+                        list_add (&bl->list, granted);
+                }
+        }
+        return;
+}
+
+/* Grant all reservelks blocked on lock(s) */
+void
+grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode)
+{
+        struct list_head granted;
+        posix_lock_t *lock = NULL;
+        posix_lock_t *tmp = NULL;
+
+        INIT_LIST_HEAD (&granted);
+
+        if (list_empty (&pl_inode->blocked_reservelks)) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "No blocked locks to be granted");
+                return;
+        }
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __grant_blocked_reserve_locks (this, pl_inode, &granted);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (lock, tmp, &granted, list) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Granted",
+                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                        lock->client_pid,
+                        lkowner_utoa (&lock->owner),
+                        lock->user_flock.l_start,
+                        lock->user_flock.l_len);
+
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0, &lock->user_flock,
+                                     NULL);
+        }
+
+}
+
+static void
+__grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode,
+                            struct list_head *granted)
+{
+        int              bl_ret = 0;
+        posix_lock_t *bl = NULL;
+        posix_lock_t *tmp = NULL;
+
+        struct list_head blocked_list;
+
+        INIT_LIST_HEAD (&blocked_list);
+        list_splice_init (&pl_inode->blocked_reservelks, &blocked_list);
+
+        list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
+
+                list_del_init (&bl->list);
+
+                bl_ret = pl_verify_reservelk (this, pl_inode, bl, bl->blocked);
+
+                if (bl_ret == 0) {
+                        list_add_tail (&bl->list, granted);
+                }
+        }
+        return;
+}
+
+void
+grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode)
+{
+        struct list_head granted;
+        posix_lock_t *lock = NULL;
+        posix_lock_t *tmp = NULL;
+        fd_t         *fd  = NULL;
+
+        int can_block = 0;
+        int32_t cmd = 0;
+        int ret = 0;
+
+        if (list_empty (&pl_inode->blocked_calls)) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "No blocked lock calls to be granted");
+                return;
+        }
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __grant_blocked_lock_calls (this, pl_inode, &granted);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (lock, tmp, &granted, list) {
+                fd = fd_from_fdnum (lock);
+
+                if (lock->blocked) {
+                        can_block = 1;
+                        cmd = F_SETLKW;
+                }
+                else
+                        cmd = F_SETLK;
+
+                lock->blocked = 0;
+                ret = pl_setlk (this, pl_inode, lock, can_block);
+                if (ret == -1) {
+                        if (can_block) {
+                                pl_trace_block (this, lock->frame, fd, NULL,
+                                                cmd, &lock->user_flock, NULL);
+                                continue;
+                        } else {
+                                gf_log (this->name, GF_LOG_DEBUG, "returning EAGAIN");
+                                pl_trace_out (this, lock->frame, fd, NULL, cmd,
+                                              &lock->user_flock, -1, EAGAIN, NULL);
+                                pl_update_refkeeper (this, fd->inode);
+                                STACK_UNWIND_STRICT (lk, lock->frame, -1,
+                                                     EAGAIN, &lock->user_flock,
+                                                     NULL);
+                                __destroy_lock (lock);
+                        }
+                }
+
+        }
+
+}
+
+
+int
+pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock)
+{
+        posix_lock_t *retlock = NULL;
+        int ret = -1;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                retlock = __reserve_unlock_lock (this, lock, pl_inode);
+                if (!retlock) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Bad Unlock issued on Inode lock");
+                        ret = -EINVAL;
+                        goto out;
+                }
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Reservelk Unlock successful");
+                __destroy_reserve_lock (retlock);
+                ret = 0;
+        }
+out:
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        grant_blocked_reserve_locks (this, pl_inode);
+        grant_blocked_lock_calls (this, pl_inode);
+
+        return ret;
+
+}
+
+int
+pl_reserve_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                  int can_block)
+{
+        int ret = -EINVAL;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+
+                        ret = __lock_reservelk (this, pl_inode, lock, can_block);
+                        if (ret < 0)
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
+                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                        lock->client_pid,
+                                        lkowner_utoa (&lock->owner),
+                                        lock->user_flock.l_start,
+                                        lock->user_flock.l_len);
+                        else
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
+                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                        lock->client_pid,
+                                        lkowner_utoa (&lock->owner),
+                                        lock->fl_start,
+                                        lock->fl_end);
+
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+        return ret;
+}
diff --git a/xlators/features/locks/tests/unit-test.c b/xlators/features/locks/tests/unit-test.c
index 6a1bfbf68..d2cca32de 100644
--- a/xlators/features/locks/tests/unit-test.c
+++ b/xlators/features/locks/tests/unit-test.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/storage/bdb/Makefile.am b/xlators/features/mac-compat/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/xlators/storage/bdb/Makefile.am
+++ b/xlators/features/mac-compat/Makefile.am
diff --git a/xlators/features/mac-compat/src/Makefile.am b/xlators/features/mac-compat/src/Makefile.am
new file mode 100644
index 000000000..f8567edce
--- /dev/null
+++ b/xlators/features/mac-compat/src/Makefile.am
@@ -0,0 +1,14 @@
+xlator_LTLIBRARIES = mac-compat.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+mac_compat_la_LDFLAGS = -module -avoid-version
+
+mac_compat_la_SOURCES = mac-compat.c
+mac_compat_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES = 
+
diff --git a/xlators/features/mac-compat/src/mac-compat.c b/xlators/features/mac-compat/src/mac-compat.c
new file mode 100644
index 000000000..7cb550ad5
--- /dev/null
+++ b/xlators/features/mac-compat/src/mac-compat.c
@@ -0,0 +1,237 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "compat-errno.h"
+
+
+enum apple_xattr {
+        GF_FINDER_INFO_XATTR,
+        GF_RESOURCE_FORK_XATTR,
+        GF_XATTR_ALL,
+        GF_XATTR_NONE
+};
+
+static char *apple_xattr_name[] = {
+        [GF_FINDER_INFO_XATTR]   = "com.apple.FinderInfo",
+        [GF_RESOURCE_FORK_XATTR] = "com.apple.ResourceFork"
+};
+
+static const char *apple_xattr_value[] = {
+        [GF_FINDER_INFO_XATTR]   =
+        /* 1 2 3 4 5 6 7 8 */
+         "\0\0\0\0\0\0\0\0"
+         "\0\0\0\0\0\0\0\0"
+         "\0\0\0\0\0\0\0\0"
+         "\0\0\0\0\0\0\0\0",
+        [GF_RESOURCE_FORK_XATTR] = ""
+};
+
+static int32_t apple_xattr_len[] = {
+        [GF_FINDER_INFO_XATTR]   = 32,
+        [GF_RESOURCE_FORK_XATTR] = 1
+};
+
+
+int32_t
+maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
+{
+        intptr_t ax = (intptr_t)this->private;
+        int i = 0;
+
+        if ((ax == GF_XATTR_ALL && op_ret >= 0) || ax != GF_XATTR_NONE) {
+                op_ret = op_errno = 0;
+
+                for (i = 0; i < GF_XATTR_ALL; i++) {
+                        if (dict_get (dict, apple_xattr_name[i]))
+                                continue;
+
+                        if (dict_set (dict, apple_xattr_name[i],
+                                      bin_to_data ((void *)apple_xattr_value[i],
+                                                   apple_xattr_len[i])) == -1) {
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+
+                                break;
+                        }
+                }
+         }
+
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+
+        return 0;
+}
+
+
+int32_t
+maccomp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
+{
+        intptr_t ax = GF_XATTR_NONE;
+        int i = 0;
+
+        if (name) {
+                for (i = 0; i < GF_XATTR_ALL; i++) {
+                        if (strcmp (apple_xattr_name[i], name) == 0) {
+                                ax = i;
+
+                                break;
+                        }
+                }
+        } else
+                ax = GF_XATTR_ALL;
+
+        this->private = (void *)ax;
+
+        STACK_WIND (frame, maccomp_getxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+
+int32_t
+maccomp_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        intptr_t ax = GF_XATTR_NONE;
+        int i = 0;
+
+        if (name) {
+                for (i = 0; i < GF_XATTR_ALL; i++) {
+                        if (strcmp (apple_xattr_name[i], name) == 0) {
+                                ax = i;
+
+                                break;
+                        }
+                }
+        } else
+                ax = GF_XATTR_ALL;
+
+        this->private = (void *)ax;
+
+        STACK_WIND (frame, maccomp_getxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+
+int32_t
+maccomp_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        intptr_t ax = (intptr_t)this->private;
+
+        if (op_ret == -1 && ax != GF_XATTR_NONE)
+                op_ret = op_errno = 0;
+
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+
+int32_t
+maccomp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                  int32_t flags, dict_t *xdata)
+{
+        intptr_t ax = GF_XATTR_NONE;
+        int i = 0;
+
+        for (i = 0; i < GF_XATTR_ALL; i++) {
+                if (dict_get (dict, apple_xattr_name[i])) {
+                        ax = i;
+
+                        break;
+                }
+        }
+
+        this->private = (void *)ax;
+
+        STACK_WIND (frame, maccomp_setxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+        return 0;
+}
+
+
+int32_t
+maccomp_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                   int32_t flags, dict_t *xdata)
+{
+        intptr_t ax = GF_XATTR_NONE;
+        int i = 0;
+
+        for (i = 0; i < GF_XATTR_ALL; i++) {
+                if (dict_get (dict, apple_xattr_name[i])) {
+                        ax = i;
+
+                        break;
+                }
+        }
+
+        this->private = (void *)ax;
+
+        STACK_WIND (frame, maccomp_setxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
+}
+
+
+int32_t
+init (xlator_t *this)
+{
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "translator not configured with exactly one child");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+        return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        return;
+}
+
+
+struct xlator_fops fops = {
+        .getxattr    = maccomp_getxattr,
+        .fgetxattr   = maccomp_fgetxattr,
+        .setxattr    = maccomp_setxattr,
+        .fsetxattr   = maccomp_fsetxattr,
+};
+
+struct xlator_cbks cbks;
+
+struct volume_options options[] = {
+        { .key = {NULL} },
+};
diff --git a/xlators/features/marker/Makefile.am b/xlators/features/marker/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/marker/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/marker/src/Makefile.am b/xlators/features/marker/src/Makefile.am
new file mode 100644
index 000000000..a7c676472
--- /dev/null
+++ b/xlators/features/marker/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = marker.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+marker_la_LDFLAGS = -module -avoid-version
+
+marker_la_SOURCES = marker.c marker-quota.c marker-quota-helper.c marker-common.c
+marker_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = marker-mem-types.h marker.h marker-quota.h marker-quota-helper.h marker-common.h $(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/lib/src
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+
+CLEANFILES =
+
diff --git a/xlators/features/marker/src/marker-common.c b/xlators/features/marker/src/marker-common.c
new file mode 100644
index 000000000..84a718add
--- /dev/null
+++ b/xlators/features/marker/src/marker-common.c
@@ -0,0 +1,69 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <fnmatch.h>
+#include "marker-common.h"
+
+marker_inode_ctx_t *
+marker_inode_ctx_new ()
+{
+        marker_inode_ctx_t *ctx = NULL;
+
+        ctx = GF_CALLOC (1, sizeof (marker_inode_ctx_t),
+                         gf_marker_mt_marker_inode_ctx_t);
+        if (ctx == NULL)
+                goto out;
+
+        ctx->quota_ctx = NULL;
+out:
+        return ctx;
+}
+
+int32_t
+marker_force_inode_ctx_get (inode_t *inode, xlator_t *this,
+                            marker_inode_ctx_t **ctx)
+{
+        int32_t  ret     = -1;
+        uint64_t ctx_int = 0;
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &ctx_int);
+                if (ret == 0)
+                        *ctx = (marker_inode_ctx_t *) (unsigned long)ctx_int;
+                else {
+                        *ctx = marker_inode_ctx_new ();
+                        if (*ctx == NULL)
+                                goto unlock;
+
+                        ret = __inode_ctx_put (inode, this,
+                                               (uint64_t )(unsigned long) *ctx);
+                        if (ret == -1) {
+                                GF_FREE (*ctx);
+                                goto unlock;
+                        }
+                        ret = 0;
+                }
+        }
+unlock: UNLOCK (&inode->lock);
+
+        return ret;
+}
+
+int
+marker_filter_quota_xattr (dict_t *dict, char *key,
+                           data_t *value, void *data)
+{
+        dict_del (dict, key);
+        return 0;
+}
diff --git a/xlators/features/marker/src/marker-common.h b/xlators/features/marker/src/marker-common.h
new file mode 100644
index 000000000..23dd846cb
--- /dev/null
+++ b/xlators/features/marker/src/marker-common.h
@@ -0,0 +1,27 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _MARKER_COMMON_H
+#define _MARKER_COMMON_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "inode.h"
+#include "xlator.h"
+#include "marker.h"
+
+int32_t
+marker_force_inode_ctx_get (inode_t *, xlator_t *, marker_inode_ctx_t **);
+
+int
+marker_filter_quota_xattr (dict_t *, char *, data_t *, void *);
+#endif
diff --git a/xlators/features/marker/src/marker-mem-types.h b/xlators/features/marker/src/marker-mem-types.h
new file mode 100644
index 000000000..1f74d5048
--- /dev/null
+++ b/xlators/features/marker/src/marker-mem-types.h
@@ -0,0 +1,25 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __MARKER_MEM_TYPES_H__
+#define __MARKER_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_marker_mem_types_ {
+        gf_marker_mt_marker_conf_t = gf_common_mt_end + 1,
+        gf_marker_mt_loc_t,
+        gf_marker_mt_volume_mark,
+        gf_marker_mt_int64_t,
+        gf_marker_mt_quota_inode_ctx_t,
+        gf_marker_mt_marker_inode_ctx_t,
+        gf_marker_mt_inode_contribution_t,
+        gf_marker_mt_end
+};
+#endif
diff --git a/xlators/features/marker/src/marker-quota-helper.c b/xlators/features/marker/src/marker-quota-helper.c
new file mode 100644
index 000000000..af5fed132
--- /dev/null
+++ b/xlators/features/marker/src/marker-quota-helper.c
@@ -0,0 +1,414 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "locking.h"
+#include "marker-quota.h"
+#include "marker-common.h"
+#include "marker-quota-helper.h"
+#include "marker-mem-types.h"
+
+int
+mq_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+{
+        int ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", loc, out);
+        GF_VALIDATE_OR_GOTO ("marker", inode, out);
+        GF_VALIDATE_OR_GOTO ("marker", path, out);
+        /* Not checking for parent because while filling
+         * loc of root, parent will be NULL
+         */
+
+        if (inode) {
+                loc->inode = inode_ref (inode);
+        }
+
+        if (parent)
+                loc->parent = inode_ref (parent);
+
+        loc->path = gf_strdup (path);
+        if (!loc->path) {
+                gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
+                goto loc_wipe;
+        }
+
+        loc->name = strrchr (loc->path, '/');
+        if (loc->name)
+                loc->name++;
+        else
+                goto loc_wipe;
+
+        ret = 0;
+loc_wipe:
+        if (ret < 0)
+                loc_wipe (loc);
+out:
+        return ret;
+}
+
+
+int32_t
+mq_inode_loc_fill (const char *parent_gfid, inode_t *inode, loc_t *loc)
+{
+        char            *resolvedpath = NULL;
+        inode_t         *parent       = NULL;
+        int              ret          = -1;
+
+        if ((!inode) || (!loc))
+                return ret;
+
+        if ((inode) && __is_root_gfid (inode->gfid)) {
+                loc->parent = NULL;
+                goto ignore_parent;
+        }
+
+        if (parent_gfid == NULL)
+                parent = inode_parent (inode, 0, NULL);
+        else
+                parent = inode_find (inode->table,
+                                     (unsigned char *) parent_gfid);
+
+        if (parent == NULL)
+                goto err;
+
+ignore_parent:
+        ret = inode_path (inode, NULL, &resolvedpath);
+        if (ret < 0)
+                goto err;
+
+        ret = mq_loc_fill (loc, inode, parent, resolvedpath);
+        if (ret < 0)
+                goto err;
+
+err:
+        if (parent)
+                inode_unref (parent);
+
+        GF_FREE (resolvedpath);
+
+        return ret;
+}
+
+
+quota_inode_ctx_t *
+mq_alloc_inode_ctx ()
+{
+        int32_t                  ret    = -1;
+        quota_inode_ctx_t       *ctx    = NULL;
+
+        QUOTA_ALLOC (ctx, quota_inode_ctx_t, ret);
+        if (ret == -1)
+                goto out;
+
+        ctx->size = 0;
+        ctx->dirty = 0;
+        ctx->updation_status = _gf_false;
+        LOCK_INIT (&ctx->lock);
+        INIT_LIST_HEAD (&ctx->contribution_head);
+out:
+        return ctx;
+}
+
+inode_contribution_t *
+mq_get_contribution_node (inode_t *inode, quota_inode_ctx_t *ctx)
+{
+        inode_contribution_t    *contri = NULL;
+        inode_contribution_t    *temp   = NULL;
+
+        if (!inode || !ctx)
+                goto out;
+
+        list_for_each_entry (temp, &ctx->contribution_head, contri_list) {
+                if (uuid_compare (temp->gfid, inode->gfid) == 0) {
+                        contri = temp;
+                        goto out;
+                }
+        }
+out:
+        return contri;
+}
+
+
+int32_t
+mq_delete_contribution_node (dict_t *dict, char *key,
+                             inode_contribution_t *contribution)
+{
+        if (dict_get (dict, key) != NULL)
+                goto out;
+
+        QUOTA_FREE_CONTRIBUTION_NODE (contribution);
+out:
+        return 0;
+}
+
+
+inode_contribution_t *
+__mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx, loc_t *loc)
+{
+        int32_t ret = 0;
+        inode_contribution_t *contribution = NULL;
+
+        if (!loc->parent) {
+                if (!uuid_is_null (loc->pargfid))
+                        loc->parent = inode_find (loc->inode->table,
+                                                  loc->pargfid);
+                if (!loc->parent)
+                        loc->parent = inode_parent (loc->inode, loc->pargfid,
+                                                    loc->name);
+                if (!loc->parent)
+                        goto out;
+        }
+
+        list_for_each_entry (contribution, &ctx->contribution_head, contri_list) {
+                if (loc->parent &&
+                     uuid_compare (contribution->gfid, loc->parent->gfid) == 0) {
+                        goto out;
+                }
+        }
+
+        QUOTA_ALLOC (contribution, inode_contribution_t, ret);
+        if (ret == -1)
+                goto out;
+
+        contribution->contribution = 0;
+
+        uuid_copy (contribution->gfid, loc->parent->gfid);
+
+	LOCK_INIT (&contribution->lock);
+        INIT_LIST_HEAD (&contribution->contri_list);
+
+        list_add_tail (&contribution->contri_list, &ctx->contribution_head);
+
+out:
+        return contribution;
+}
+
+
+inode_contribution_t *
+mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx, loc_t *loc)
+{
+        inode_contribution_t *contribution = NULL;
+
+        if ((ctx == NULL) || (loc == NULL))
+                return NULL;
+
+        if (strcmp (loc->path, "/") == 0)
+                return NULL;
+
+        LOCK (&ctx->lock);
+        {
+                contribution = __mq_add_new_contribution_node (this, ctx, loc);
+        }
+        UNLOCK (&ctx->lock);
+
+        return contribution;
+}
+
+
+int32_t
+mq_dict_set_contribution (xlator_t *this, dict_t *dict,
+                          loc_t *loc)
+{
+        int32_t ret              = -1;
+        char    contri_key [512] = {0, };
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", dict, out);
+        GF_VALIDATE_OR_GOTO ("marker", loc, out);
+        GF_VALIDATE_OR_GOTO ("marker", loc->parent, out);
+
+        GET_CONTRI_KEY (contri_key, loc->parent->gfid, ret);
+        if (ret < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_int64 (dict, contri_key, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "unable to set dict value on %s.",
+                        loc->path);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int32_t
+mq_inode_ctx_get (inode_t *inode, xlator_t *this,
+                  quota_inode_ctx_t **ctx)
+{
+        int32_t             ret      = -1;
+        uint64_t            ctx_int  = 0;
+        marker_inode_ctx_t *mark_ctx = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", inode, out);
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+        if (ret < 0) {
+                ret = -1;
+                *ctx = NULL;
+                goto out;
+        }
+
+        mark_ctx = (marker_inode_ctx_t *) (unsigned long)ctx_int;
+        if (mark_ctx->quota_ctx == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        *ctx = mark_ctx->quota_ctx;
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+quota_inode_ctx_t *
+__mq_inode_ctx_new (inode_t *inode, xlator_t *this)
+{
+        int32_t               ret        = -1;
+        quota_inode_ctx_t    *quota_ctx  = NULL;
+        marker_inode_ctx_t   *mark_ctx   = NULL;
+
+        ret = marker_force_inode_ctx_get (inode, this, &mark_ctx);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "marker_force_inode_ctx_get() failed");
+                goto out;
+        }
+
+        LOCK (&inode->lock);
+        {
+                if (mark_ctx->quota_ctx == NULL) {
+                        quota_ctx = mq_alloc_inode_ctx ();
+                        if (quota_ctx == NULL) {
+                                ret = -1;
+                                goto unlock;
+                        }
+                        mark_ctx->quota_ctx = quota_ctx;
+                } else {
+                        quota_ctx = mark_ctx->quota_ctx;
+                }
+
+                ret = 0;
+        }
+unlock:
+        UNLOCK (&inode->lock);
+out:
+        return quota_ctx;
+}
+
+
+quota_inode_ctx_t *
+mq_inode_ctx_new (inode_t * inode, xlator_t *this)
+{
+        return __mq_inode_ctx_new (inode, this);
+}
+
+quota_local_t *
+mq_local_new ()
+{
+        quota_local_t  *local   = NULL;
+
+        local = mem_get0 (THIS->local_pool);
+        if (!local)
+                goto out;
+
+        local->ref = 1;
+        LOCK_INIT (&local->lock);
+
+        local->ctx = NULL;
+        local->contri = NULL;
+
+out:
+        return local;
+}
+
+quota_local_t *
+mq_local_ref (quota_local_t *local)
+{
+        LOCK (&local->lock);
+        {
+                local->ref ++;
+        }
+        UNLOCK (&local->lock);
+
+        return local;
+}
+
+
+int32_t
+mq_local_unref (xlator_t *this, quota_local_t *local)
+{
+        int32_t ref = 0;
+        if (local == NULL)
+                goto out;
+
+        QUOTA_SAFE_DECREMENT (&local->lock, local->ref, ref);
+
+        if (ref != 0)
+                goto out;
+
+        if (local->fd != NULL)
+                fd_unref (local->fd);
+
+        loc_wipe (&local->loc);
+
+        loc_wipe (&local->parent_loc);
+
+        LOCK_DESTROY (&local->lock);
+
+        mem_put (local);
+out:
+        return 0;
+}
+
+
+inode_contribution_t *
+mq_get_contribution_from_loc (xlator_t *this, loc_t *loc)
+{
+        int32_t               ret          = 0;
+        quota_inode_ctx_t    *ctx          = NULL;
+        inode_contribution_t *contribution = NULL;
+
+        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
+        if (ret < 0) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                                  "cannot get marker-quota context from inode "
+                                  "(gfid:%s, path:%s)",
+                                  uuid_utoa (loc->inode->gfid), loc->path);
+                goto err;
+        }
+
+        contribution = mq_get_contribution_node (loc->parent, ctx);
+        if (contribution == NULL) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                                  "inode (gfid:%s, path:%s) has "
+                                  "no contribution towards parent (gfid:%s)",
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, uuid_utoa (loc->parent->gfid));
+                goto err;
+        }
+
+err:
+        return contribution;
+}
diff --git a/xlators/features/marker/src/marker-quota-helper.h b/xlators/features/marker/src/marker-quota-helper.h
new file mode 100644
index 000000000..6cdd14881
--- /dev/null
+++ b/xlators/features/marker/src/marker-quota-helper.h
@@ -0,0 +1,76 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _MARKER_QUOTA_HELPER_H
+#define _MARKER_QUOTA_HELPER
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "marker.h"
+
+#define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)     \
+        do {                                            \
+                list_del (&_contribution->contri_list); \
+                GF_FREE (_contribution);                \
+        } while (0)
+
+#define QUOTA_SAFE_INCREMENT(lock, var)                 \
+        do {                                            \
+                LOCK (lock);                            \
+                        var ++;                         \
+                UNLOCK (lock);                          \
+        } while (0)
+
+#define QUOTA_SAFE_DECREMENT(lock, var, value)  \
+        do {                                    \
+                LOCK (lock);                    \
+                {                               \
+                      value = --var;            \
+                }                               \
+                UNLOCK (lock);                  \
+        } while (0)
+
+inode_contribution_t *
+mq_add_new_contribution_node (xlator_t *, quota_inode_ctx_t *, loc_t *);
+
+int32_t
+mq_dict_set_contribution (xlator_t *, dict_t *, loc_t *);
+
+quota_inode_ctx_t *
+mq_inode_ctx_new (inode_t *, xlator_t *);
+
+int32_t
+mq_inode_ctx_get (inode_t *, xlator_t *, quota_inode_ctx_t **);
+
+int32_t
+mq_delete_contribution_node (dict_t *, char *, inode_contribution_t *);
+
+int32_t
+mq_inode_loc_fill (const char *, inode_t *, loc_t *);
+
+quota_local_t *
+mq_local_new ();
+
+quota_local_t *
+mq_local_ref (quota_local_t *);
+
+int32_t
+mq_local_unref (xlator_t *, quota_local_t *);
+
+inode_contribution_t *
+mq_get_contribution_node (inode_t *, quota_inode_ctx_t *);
+
+inode_contribution_t *
+mq_get_contribution_from_loc (xlator_t *this, loc_t *loc);
+
+#endif
diff --git a/xlators/features/marker/src/marker-quota.c b/xlators/features/marker/src/marker-quota.c
new file mode 100644
index 000000000..6f9af6e13
--- /dev/null
+++ b/xlators/features/marker/src/marker-quota.c
@@ -0,0 +1,2520 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "dict.h"
+#include "xlator.h"
+#include "defaults.h"
+#include "libxlator.h"
+#include "common-utils.h"
+#include "byte-order.h"
+#include "marker-quota.h"
+#include "marker-quota-helper.h"
+
+int
+mq_loc_copy (loc_t *dst, loc_t *src)
+{
+        int ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", dst, out);
+        GF_VALIDATE_OR_GOTO ("marker", src, out);
+
+        if (src->inode == NULL ||
+            src->path == NULL) {
+                gf_log ("marker", GF_LOG_WARNING,
+                        "src loc is not valid");
+                goto out;
+        }
+
+        ret = loc_copy (dst, src);
+out:
+        return ret;
+}
+
+int32_t
+mq_get_local_err (quota_local_t *local,
+                  int32_t *val)
+{
+        int32_t ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", local, out);
+        GF_VALIDATE_OR_GOTO ("marker", val, out);
+
+        LOCK (&local->lock);
+        {
+                *val = local->err;
+        }
+        UNLOCK (&local->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+mq_get_ctx_updation_status (quota_inode_ctx_t *ctx,
+                            gf_boolean_t *status)
+{
+        int32_t   ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+        GF_VALIDATE_OR_GOTO ("marker", status, out);
+
+        LOCK (&ctx->lock);
+        {
+                *status = ctx->updation_status;
+        }
+        UNLOCK (&ctx->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int32_t
+mq_set_ctx_updation_status (quota_inode_ctx_t *ctx,
+                            gf_boolean_t status)
+{
+        int32_t   ret = -1;
+
+        if (ctx == NULL)
+                goto out;
+
+        LOCK (&ctx->lock);
+        {
+                ctx->updation_status = status;
+        }
+        UNLOCK (&ctx->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+mq_test_and_set_ctx_updation_status (quota_inode_ctx_t *ctx,
+                                     gf_boolean_t *status)
+{
+        int32_t         ret     = -1;
+        gf_boolean_t    temp    = _gf_false;
+
+        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+        GF_VALIDATE_OR_GOTO ("marker", status, out);
+
+        LOCK (&ctx->lock);
+        {
+                temp = *status;
+                *status = ctx->updation_status;
+                ctx->updation_status = temp;
+        }
+        UNLOCK (&ctx->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+mq_assign_lk_owner (xlator_t *this, call_frame_t *frame)
+{
+        marker_conf_t *conf     = NULL;
+        uint64_t       lk_owner = 0;
+
+        conf = this->private;
+
+        LOCK (&conf->lock);
+        {
+                if (++conf->quota_lk_owner == 0) {
+                        ++conf->quota_lk_owner;
+                }
+
+                lk_owner = conf->quota_lk_owner;
+        }
+        UNLOCK (&conf->lock);
+
+        set_lk_owner_from_uint64 (&frame->root->lk_owner, lk_owner);
+
+        return;
+}
+
+
+int32_t
+mq_loc_fill_from_name (xlator_t *this, loc_t *newloc, loc_t *oldloc,
+                       uint64_t ino, char *name)
+{
+        int32_t   ret  = -1;
+        int32_t   len  = 0;
+        char     *path = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", newloc, out);
+        GF_VALIDATE_OR_GOTO ("marker", oldloc, out);
+        GF_VALIDATE_OR_GOTO ("marker", name, out);
+
+        newloc->inode = inode_new (oldloc->inode->table);
+
+        if (!newloc->inode) {
+                ret = -1;
+                goto out;
+        }
+
+        newloc->parent = inode_ref (oldloc->inode);
+        uuid_copy (newloc->pargfid, oldloc->inode->gfid);
+
+        len = strlen (oldloc->path);
+
+        if (oldloc->path [len - 1] == '/')
+                ret = gf_asprintf ((char **) &path, "%s%s",
+                                   oldloc->path, name);
+        else
+                ret = gf_asprintf ((char **) &path, "%s/%s",
+                                   oldloc->path, name);
+
+        if (ret < 0)
+                goto out;
+
+        newloc->path = path;
+
+        newloc->name = strrchr (newloc->path, '/');
+
+        if (newloc->name)
+                newloc->name++;
+
+        gf_log (this->name, GF_LOG_DEBUG, "path = %s name =%s",
+                newloc->path, newloc->name);
+out:
+        return ret;
+}
+
+int32_t
+mq_dirty_inode_updation_done (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        QUOTA_STACK_DESTROY (frame, this);
+
+        return 0;
+}
+
+int32_t
+mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *this,
+                                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        struct gf_flock   lock  = {0, };
+        quota_local_t    *local = NULL;
+        loc_t             loc = {0, };
+	int               ret = -1;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                local->err = -1;
+
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
+
+                return 0;
+        }
+
+        if (op_ret == 0)
+                local->ctx->dirty = 0;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+        ret = loc_copy (&loc, &local->loc);
+	if (ret == -1) {
+                local->err = -1;
+                frame->local = NULL;
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
+                return 0;
+        }
+
+        if (local->loc.inode == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Inode is NULL, so can't stackwind.");
+                goto out;
+        }
+
+        STACK_WIND (frame,
+                    mq_dirty_inode_updation_done,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &loc, F_SETLKW, &lock, NULL);
+
+        loc_wipe (&loc);
+
+        return 0;
+out:
+        mq_dirty_inode_updation_done (frame, NULL, this, -1, 0, NULL);
+
+        return 0;
+}
+
+int32_t
+mq_mark_inode_undirty (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
+{
+        int32_t        ret     = -1;
+        int64_t       *size    = NULL;
+        dict_t        *newdict = NULL;
+        quota_local_t *local   = NULL;
+
+        local = (quota_local_t *) frame->local;
+
+        if (op_ret == -1)
+                goto err;
+
+        if (!dict)
+                goto wind;
+
+        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+        if (ret)
+                goto wind;
+
+        LOCK (&local->ctx->lock);
+        {
+                local->ctx->size = ntoh64 (*size);
+        }
+        UNLOCK (&local->ctx->lock);
+
+wind:
+        newdict = dict_new ();
+        if (!newdict)
+                goto err;
+
+        ret = dict_set_int8 (newdict, QUOTA_DIRTY_KEY, 0);
+        if (ret)
+                goto err;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+        STACK_WIND (frame, mq_release_lock_on_dirty_inode,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr,
+                    &local->loc, newdict, 0, NULL);
+        ret = 0;
+
+err:
+        if (op_ret == -1 || ret == -1) {
+                local->err = -1;
+
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (newdict)
+                dict_unref (newdict);
+
+        return 0;
+}
+
+int32_t
+mq_update_size_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, dict_t *dict, struct iatt *postparent)
+{
+        int32_t          ret      = -1;
+        dict_t          *new_dict = NULL;
+        int64_t         *size     = NULL;
+        int64_t         *delta    = NULL;
+        quota_local_t   *local    = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1)
+                goto err;
+
+        if (dict == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Dict is null while updating the size xattr %s",
+                        local->loc.path?local->loc.path:"");
+                goto err;
+        }
+
+        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+        if (!size) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to get the size, %s",
+                        local->loc.path?local->loc.path:"");
+                goto err;
+        }
+
+        QUOTA_ALLOC_OR_GOTO (delta, int64_t, ret, err);
+
+        *delta = hton64 (local->sum - ntoh64 (*size));
+
+        gf_log (this->name, GF_LOG_DEBUG, "calculated size = %"PRId64", "
+                "original size = %"PRIu64
+                " path = %s diff = %"PRIu64, local->sum, ntoh64 (*size),
+                local->loc.path, ntoh64 (*delta));
+
+        new_dict = dict_new ();
+        if (!new_dict);
+
+        ret = dict_set_bin (new_dict, QUOTA_SIZE_KEY, delta, 8);
+        if (ret)
+                goto err;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame, mq_mark_inode_undirty, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop, &local->loc,
+                    GF_XATTROP_ADD_ARRAY64, new_dict, NULL);
+
+        ret = 0;
+
+err:
+        if (op_ret == -1 || ret == -1) {
+                local->err = -1;
+
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (new_dict)
+                dict_unref (new_dict);
+
+        return 0;
+}
+
+int32_t
+mq_test_and_set_local_err(quota_local_t *local,
+                          int32_t *val)
+{
+        int     tmp = 0;
+        int32_t ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", local, out);
+        GF_VALIDATE_OR_GOTO ("marker", val, out);
+
+        LOCK (&local->lock);
+        {
+                tmp = local->err;
+                local->err = *val;
+                *val = tmp;
+        }
+        UNLOCK (&local->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+mq_get_dirty_inode_size (call_frame_t *frame, xlator_t *this)
+{
+        int32_t        ret   = -1;
+        dict_t        *dict  = NULL;
+        quota_local_t *local = NULL;
+
+        local = (quota_local_t *) frame->local;
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto err;
+        }
+
+        ret = dict_set_int64 (dict, QUOTA_SIZE_KEY, 0);
+        if (ret)
+                goto err;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame, mq_update_size_xattr, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &local->loc, dict);
+        ret =0;
+
+err:
+        if (ret) {
+                local->err = -1;
+
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (dict)
+                dict_unref (dict);
+
+        return 0;
+}
+
+int32_t
+mq_get_child_contribution (call_frame_t *frame,
+                           void *cookie,
+                           xlator_t *this,
+                           int32_t op_ret,
+                           int32_t op_errno,
+                           inode_t *inode,
+                           struct iatt *buf,
+                           dict_t *dict,
+                           struct iatt *postparent)
+{
+        int32_t        ret                = -1;
+        int32_t        val                = 0;
+        char           contri_key [512]   = {0, };
+        int64_t       *contri             = NULL;
+        quota_local_t *local              = NULL;
+
+        local = frame->local;
+
+        frame->local = NULL;
+
+        QUOTA_STACK_DESTROY (frame, this);
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        strerror (op_errno));
+                val = -2;
+                if (!mq_test_and_set_local_err (local, &val) &&
+                    val != -2)
+                        mq_release_lock_on_dirty_inode (local->frame, NULL,
+                                                        this, 0, 0, NULL);
+
+                goto exit;
+        }
+
+        ret = mq_get_local_err (local, &val);
+        if (!ret && val == -2)
+                goto exit;
+
+        GET_CONTRI_KEY (contri_key, local->loc.inode->gfid, ret);
+        if (ret < 0)
+                goto out;
+
+        if (!dict)
+                goto out;
+
+        if (dict_get_bin (dict, contri_key, (void **) &contri) == 0)
+                local->sum += ntoh64 (*contri);
+
+out:
+        LOCK (&local->lock);
+        {
+                val = --local->dentry_child_count;
+        }
+        UNLOCK (&local->lock);
+
+        if (val == 0) {
+                mq_dirty_inode_readdir (local->frame, NULL, this,
+                                        0, 0, NULL, NULL);
+        }
+        mq_local_unref (this, local);
+
+        return 0;
+exit:
+        mq_local_unref (this, local);
+        return 0;
+}
+
+int32_t
+mq_readdir_cbk (call_frame_t *frame,
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                gf_dirent_t *entries, dict_t *xdata)
+{
+        char           contri_key [512]   = {0, };
+        int32_t        ret                = 0;
+        int32_t        val                = 0;
+        off_t          offset             = 0;
+        int32_t        count              = 0;
+        dict_t        *dict               = NULL;
+        quota_local_t *local              = NULL;
+        gf_dirent_t   *entry              = NULL;
+        call_frame_t  *newframe           = NULL;
+        loc_t          loc                = {0, };
+
+        local = mq_local_ref (frame->local);
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "readdir failed %s", strerror (op_errno));
+                local->err = -1;
+
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+
+                goto end;
+        } else if (op_ret == 0) {
+                mq_get_dirty_inode_size (frame, this);
+
+                goto end;
+        }
+
+        local->dentry_child_count =  0;
+
+        list_for_each_entry (entry, (&entries->list), list) {
+                gf_log (this->name, GF_LOG_DEBUG, "entry  = %s", entry->d_name);
+
+                if ((!strcmp (entry->d_name, ".")) || (!strcmp (entry->d_name,
+                                                                ".."))) {
+                        gf_log (this->name, GF_LOG_DEBUG, "entry  = %s",
+                                entry->d_name);
+                        continue;
+                }
+
+                offset = entry->d_off;
+                count++;
+        }
+
+        if (count == 0) {
+                mq_get_dirty_inode_size (frame, this);
+                goto end;
+
+        }
+
+        local->frame = frame;
+
+        LOCK (&local->lock);
+        {
+                local->dentry_child_count = count;
+                local->d_off = offset;
+        }
+        UNLOCK (&local->lock);
+
+
+        list_for_each_entry (entry, (&entries->list), list) {
+                gf_log (this->name, GF_LOG_DEBUG, "entry  = %s", entry->d_name);
+
+                if ((!strcmp (entry->d_name, ".")) || (!strcmp (entry->d_name,
+                                                                ".."))) {
+                        gf_log (this->name, GF_LOG_DEBUG, "entry  = %s",
+                                entry->d_name);
+                        continue;
+                }
+
+                ret = mq_loc_fill_from_name (this, &loc, &local->loc,
+                                             entry->d_ino, entry->d_name);
+                if (ret < 0)
+                        goto out;
+
+                ret = 0;
+
+                LOCK (&local->lock);
+                {
+                        if (local->err != -2) {
+                                newframe = copy_frame (frame);
+                                if (!newframe) {
+                                        ret = -1;
+                                }
+                        } else
+                                ret = -1;
+                }
+                UNLOCK (&local->lock);
+
+                if (ret == -1)
+                        goto out;
+
+                newframe->local = mq_local_ref (local);
+
+                dict = dict_new ();
+                if (!dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                GET_CONTRI_KEY (contri_key, local->loc.inode->gfid, ret);
+                if (ret < 0)
+                        goto out;
+
+                ret = dict_set_int64 (dict, contri_key, 0);
+                if (ret)
+                        goto out;
+
+                STACK_WIND (newframe,
+                            mq_get_child_contribution,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->lookup,
+                            &loc, dict);
+
+                offset = entry->d_off;
+
+                loc_wipe (&loc);
+
+                newframe = NULL;
+
+        out:
+                if (dict) {
+                        dict_unref (dict);
+                        dict = NULL;
+                }
+
+                if (ret) {
+                        val = -2;
+                        mq_test_and_set_local_err (local, &val);
+
+                        if (newframe) {
+                                newframe->local = NULL;
+                                mq_local_unref(this, local);
+                                QUOTA_STACK_DESTROY (newframe, this);
+                        }
+
+                        break;
+                }
+        }
+
+        if (ret && val != -2) {
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+end:
+        mq_local_unref (this, local);
+
+        return 0;
+}
+
+int32_t
+mq_dirty_inode_readdir (call_frame_t *frame,
+                        void *cookie,
+                        xlator_t *this,
+                        int32_t op_ret,
+                        int32_t op_errno,
+                        fd_t *fd, dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                local->err = -1;
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+                return 0;
+        }
+
+        if (local->fd == NULL)
+                local->fd = fd_ref (fd);
+
+        STACK_WIND (frame,
+                    mq_readdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir,
+                    local->fd, READDIR_BUF, local->d_off, xdata);
+
+        return 0;
+}
+
+int32_t
+mq_check_if_still_dirty (call_frame_t *frame,
+                         void *cookie,
+                         xlator_t *this,
+                         int32_t op_ret,
+                         int32_t op_errno,
+                         inode_t *inode,
+                         struct iatt *buf,
+                         dict_t *dict,
+                         struct iatt *postparent)
+{
+        int8_t           dirty          = -1;
+        int32_t          ret            = -1;
+        fd_t            *fd             = NULL;
+        quota_local_t   *local          = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to get "
+                        "the dirty xattr for %s", local->loc.path);
+                goto err;
+        }
+
+        if (!dict) {
+                ret = -1;
+                goto err;
+        }
+
+        ret = dict_get_int8 (dict, QUOTA_DIRTY_KEY, &dirty);
+        if (ret)
+                goto err;
+
+        //the inode is not dirty anymore
+        if (dirty == 0) {
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+
+                return 0;
+        }
+
+        fd = fd_create (local->loc.inode, frame->root->pid);
+
+        local->d_off = 0;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+        STACK_WIND(frame,
+                   mq_dirty_inode_readdir,
+                   FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->opendir,
+                   &local->loc, fd, NULL);
+
+        ret = 0;
+
+err:
+        if (op_ret == -1 || ret == -1) {
+                local->err = -1;
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (fd != NULL) {
+                fd_unref (fd);
+        }
+
+        return 0;
+}
+
+int32_t
+mq_get_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        int32_t        ret       = -1;
+        dict_t        *xattr_req = NULL;
+        quota_local_t *local     = NULL;
+
+        if (op_ret == -1) {
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
+                return 0;
+        }
+
+        local = frame->local;
+
+        xattr_req = dict_new ();
+        if (xattr_req == NULL) {
+                ret = -1;
+                goto err;
+        }
+
+        ret = dict_set_int8 (xattr_req, QUOTA_DIRTY_KEY, 0);
+        if (ret)
+                goto err;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame,
+                    mq_check_if_still_dirty,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    &local->loc,
+                    xattr_req);
+        ret = 0;
+
+err:
+        if (ret) {
+                local->err = -1;
+                mq_release_lock_on_dirty_inode(frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (xattr_req)
+                dict_unref (xattr_req);
+
+        return 0;
+}
+
+/* return 1 when dirty updation started
+ * 0 other wise
+ */
+int32_t
+mq_update_dirty_inode (xlator_t *this,
+                       loc_t *loc,
+                       quota_inode_ctx_t *ctx,
+                       inode_contribution_t *contribution)
+{
+        int32_t          ret        = -1;
+        quota_local_t   *local      = NULL;
+        gf_boolean_t    status     = _gf_false;
+        struct gf_flock  lock       = {0, };
+        call_frame_t    *frame      = NULL;
+
+        ret = mq_get_ctx_updation_status (ctx, &status);
+        if (ret == -1 || status == _gf_true) {
+                ret = 0;
+                goto out;
+        }
+
+        frame = create_frame (this, this->ctx->pool);
+        if (frame == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        mq_assign_lk_owner (this, frame);
+
+        local = mq_local_new ();
+        if (local == NULL)
+                goto fr_destroy;
+
+        frame->local = local;
+        ret = mq_loc_copy (&local->loc, loc);
+        if (ret < 0)
+                goto fr_destroy;
+
+        local->ctx = ctx;
+
+        local->contri = contribution;
+
+        lock.l_type = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start = 0;
+        lock.l_len = 0;
+
+        if (local->loc.inode == NULL) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Inode is NULL, so can't stackwind.");
+                goto fr_destroy;
+        }
+
+        STACK_WIND (frame,
+                    mq_get_dirty_xattr,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->loc, F_SETLKW, &lock, NULL);
+        return 1;
+
+fr_destroy:
+        QUOTA_STACK_DESTROY (frame, this);
+out:
+
+        return 0;
+}
+
+
+int32_t
+mq_inode_creation_done (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+
+        if (frame == NULL)
+                return 0;
+
+        local = frame->local;
+
+        if (local != NULL) {
+                mq_initiate_quota_txn (this, &local->loc);
+        }
+
+        QUOTA_STACK_DESTROY (frame, this);
+
+        return 0;
+}
+
+
+int32_t
+mq_xattr_creation_release_lock (call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
+{
+        struct gf_flock  lock  = {0, };
+        quota_local_t   *local = NULL;
+
+        local = frame->local;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+        STACK_WIND (frame,
+                    mq_inode_creation_done,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->loc,
+                    F_SETLKW, &lock, NULL);
+
+        return 0;
+}
+
+
+int32_t
+mq_create_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
+{
+        int32_t          ret       = -1;
+        dict_t          *newdict   = NULL;
+        quota_local_t   *local     = NULL;
+
+        if (op_ret < 0) {
+                goto err;
+        }
+
+        local = frame->local;
+
+        if (local->loc.inode->ia_type == IA_IFDIR) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        goto err;
+                }
+
+                ret = dict_set_int8 (newdict, QUOTA_DIRTY_KEY, 0);
+                if (ret == -1) {
+                        goto err;
+                }
+
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+                GF_UUID_ASSERT (local->loc.gfid);
+
+                STACK_WIND (frame, mq_xattr_creation_release_lock,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            &local->loc, newdict, 0, NULL);
+        } else {
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        ret = 0;
+
+err:
+        if (ret < 0) {
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (newdict != NULL)
+                dict_unref (newdict);
+
+        return 0;
+}
+
+
+int32_t
+mq_create_xattr (xlator_t *this, call_frame_t *frame)
+{
+        int32_t               ret       = 0;
+        int64_t              *value     = NULL;
+        int64_t              *size      = NULL;
+        dict_t               *dict      = NULL;
+        char                  key[512]  = {0, };
+        quota_local_t        *local     = NULL;
+        quota_inode_ctx_t    *ctx       = NULL;
+        inode_contribution_t *contri    = NULL;
+
+        if (frame == NULL || this == NULL)
+                return 0;
+
+        local = frame->local;
+
+        ret = mq_inode_ctx_get (local->loc.inode, this, &ctx);
+        if (ret < 0) {
+                ctx = mq_inode_ctx_new (local->loc.inode, this);
+                if (ctx == NULL) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "mq_inode_ctx_new failed");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        if (local->loc.inode->ia_type == IA_IFDIR) {
+                QUOTA_ALLOC_OR_GOTO (size, int64_t, ret, err);
+                ret = dict_set_bin (dict, QUOTA_SIZE_KEY, size, 8);
+                if (ret < 0)
+                        goto free_size;
+        }
+
+        if (strcmp (local->loc.path, "/") != 0) {
+                contri = mq_add_new_contribution_node (this, ctx, &local->loc);
+                if (contri == NULL)
+                        goto err;
+
+                QUOTA_ALLOC_OR_GOTO (value, int64_t, ret, err);
+                GET_CONTRI_KEY (key, local->loc.parent->gfid, ret);
+
+                ret = dict_set_bin (dict, key, value, 8);
+                if (ret < 0)
+                        goto free_value;
+        }
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame, mq_create_dirty_xattr, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop, &local->loc,
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
+        ret = 0;
+
+free_size:
+        if (ret < 0) {
+                GF_FREE (size);
+        }
+
+free_value:
+        if (ret < 0) {
+                GF_FREE (value);
+        }
+
+err:
+        dict_unref (dict);
+
+out:
+        if (ret < 0) {
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        return 0;
+}
+
+
+int32_t
+mq_check_n_set_inode_xattr (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            inode_t *inode, struct iatt *buf, dict_t *dict,
+                            struct iatt *postparent)
+{
+        quota_local_t        *local           = NULL;
+        int64_t              *size            = NULL, *contri = NULL;
+        int8_t                dirty           = 0;
+        int32_t               ret             = 0;
+        char                  contri_key[512] = {0, };
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+
+        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+        if (ret < 0)
+                goto create_xattr;
+
+        ret = dict_get_int8 (dict, QUOTA_DIRTY_KEY, &dirty);
+        if (ret < 0)
+                goto create_xattr;
+
+        //check contribution xattr if not root
+        if (strcmp (local->loc.path, "/") != 0) {
+                GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
+                if (ret < 0)
+                        goto out;
+
+                ret = dict_get_bin (dict, contri_key, (void **) &contri);
+                if (ret < 0)
+                        goto create_xattr;
+        }
+
+out:
+        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+        return 0;
+
+create_xattr:
+        if (uuid_is_null (local->loc.gfid)) {
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+        }
+
+        mq_create_xattr (this, frame);
+        return 0;
+}
+
+
+int32_t
+mq_get_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        dict_t        *xattr_req = NULL;
+        quota_local_t *local     = NULL;
+        int32_t        ret       = 0;
+
+        if (op_ret < 0) {
+                goto lock_err;
+        }
+
+        local = frame->local;
+
+        xattr_req = dict_new ();
+        if (xattr_req == NULL) {
+                goto err;
+        }
+
+        ret = mq_req_xattr (this, &local->loc, xattr_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "cannot request xattr");
+                goto err;
+        }
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame, mq_check_n_set_inode_xattr, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &local->loc, xattr_req);
+
+        dict_unref (xattr_req);
+
+        return 0;
+
+err:
+        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+
+        if (xattr_req)
+                dict_unref (xattr_req);
+        return 0;
+
+lock_err:
+        mq_inode_creation_done (frame, NULL, this, 0, 0, NULL);
+        return 0;
+}
+
+
+int32_t
+mq_set_inode_xattr (xlator_t *this, loc_t *loc)
+{
+        struct gf_flock  lock  = {0, };
+        quota_local_t   *local = NULL;
+        int32_t          ret   = 0;
+        call_frame_t    *frame = NULL;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto err;
+        }
+
+        local = mq_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
+
+        frame->local = local;
+
+        ret = loc_copy (&local->loc, loc);
+        if (ret < 0) {
+                goto err;
+        }
+
+        frame->local = local;
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND (frame,
+                    mq_get_xattr,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->loc, F_SETLKW, &lock, NULL);
+
+        return 0;
+
+err:
+        QUOTA_STACK_DESTROY (frame, this);
+
+        return 0;
+}
+
+
+int32_t
+mq_get_parent_inode_local (xlator_t *this, quota_local_t *local)
+{
+        int32_t            ret = -1;
+        quota_inode_ctx_t *ctx = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", local, out);
+
+        local->contri = NULL;
+
+        loc_wipe (&local->loc);
+
+        ret = mq_loc_copy (&local->loc, &local->parent_loc);
+        if (ret < 0) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                        "loc copy failed");
+                goto out;
+        }
+
+        loc_wipe (&local->parent_loc);
+
+        ret = mq_inode_loc_fill (NULL, local->loc.parent,
+                                 &local->parent_loc);
+        if (ret < 0) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                        "failed to build parent loc of %s",
+                        local->loc.path);
+                goto out;
+        }
+
+        ret = mq_inode_ctx_get (local->loc.inode, this, &ctx);
+        if (ret < 0) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                        "inode ctx get failed");
+                goto out;
+        }
+
+        local->ctx = ctx;
+
+        if (list_empty (&ctx->contribution_head)) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                        "contribution node list is empty which "
+                        "is an error");
+                ret = -1;
+                goto out;
+        }
+
+        local->contri = (inode_contribution_t *) ctx->contribution_head.next;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int32_t
+mq_xattr_updation_done (call_frame_t *frame,
+                        void *cookie,
+                        xlator_t *this,
+                        int32_t op_ret,
+                        int32_t op_errno,
+                        dict_t *dict, dict_t *xdata)
+{
+        QUOTA_STACK_DESTROY (frame, this);
+        return 0;
+}
+
+
+int32_t
+mq_inodelk_cbk (call_frame_t *frame, void *cookie,
+                xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        int32_t         ret    = 0;
+        gf_boolean_t    status = _gf_false;
+        quota_local_t  *local  = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1 || local->err) {
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "unlocking failed on path (%s)(%s)",
+                                local->parent_loc.path, strerror (op_errno));
+                }
+                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL, NULL);
+
+                return 0;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "inodelk released on %s", local->parent_loc.path);
+
+        if ((strcmp (local->parent_loc.path, "/") == 0)
+            || (local->delta == 0)) {
+                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL, NULL);
+        } else {
+                ret = mq_get_parent_inode_local (this, local);
+                if (ret < 0) {
+                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL,
+                                                NULL);
+                        goto out;
+                }
+                status = _gf_true;
+
+                ret = mq_test_and_set_ctx_updation_status (local->ctx, &status);
+                if (ret == 0 && status == _gf_false) {
+                        mq_get_lock_on_parent (frame, this);
+                } else {
+                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL,
+                                                NULL);
+                }
+        }
+out:
+        return 0;
+}
+
+
+//now release lock on the parent inode
+int32_t
+mq_release_parent_lock (call_frame_t *frame, void *cookie,
+                        xlator_t *this, int32_t op_ret,
+                        int32_t op_errno, dict_t *xdata)
+{
+        int32_t            ret      = 0;
+        quota_local_t     *local    = NULL;
+        quota_inode_ctx_t *ctx      = NULL;
+        struct gf_flock    lock     = {0, };
+
+        local = frame->local;
+
+        if (local->err != 0) {
+                gf_log_callingfn (this->name,
+                                  (local->err == ENOENT) ? GF_LOG_DEBUG
+                                  : GF_LOG_WARNING,
+                                  "An operation during quota updation "
+                                  "of path (%s) failed (%s)", local->loc.path,
+                                  strerror (local->err));
+        }
+
+        ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
+        if (ret < 0)
+                goto wind;
+
+        LOCK (&ctx->lock);
+        {
+                ctx->dirty = 0;
+        }
+        UNLOCK (&ctx->lock);
+
+        if (local->parent_loc.inode == NULL) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Invalid parent inode.");
+                goto err;
+        }
+
+wind:
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+        STACK_WIND (frame,
+                    mq_inodelk_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->parent_loc,
+                    F_SETLKW, &lock, NULL);
+
+        return 0;
+err:
+        mq_xattr_updation_done (frame, NULL, this,
+                                0, 0 , NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+mq_mark_undirty (call_frame_t *frame,
+                 void *cookie,
+                 xlator_t *this,
+                 int32_t op_ret,
+                 int32_t op_errno,
+                 dict_t *dict, dict_t *xdata)
+{
+        int32_t            ret          = -1;
+        int64_t           *size         = NULL;
+        dict_t            *newdict      = NULL;
+        quota_local_t     *local        = NULL;
+        quota_inode_ctx_t *ctx          = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "%s occurred while"
+                        " updating the size of %s", strerror (op_errno),
+                        local->parent_loc.path);
+
+                goto err;
+        }
+
+        //update the size of the parent inode
+        if (dict != NULL) {
+                ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
+                if (ret < 0) {
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
+                ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+                if (ret < 0) {
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
+                LOCK (&ctx->lock);
+                {
+                        if (size)
+                                ctx->size = ntoh64 (*size);
+                        gf_log (this->name, GF_LOG_DEBUG, "%s %"PRId64,
+                                local->parent_loc.path, ctx->size);
+                }
+                UNLOCK (&ctx->lock);
+        }
+
+        newdict = dict_new ();
+        if (!newdict) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        ret = dict_set_int8 (newdict, QUOTA_DIRTY_KEY, 0);
+
+        if (ret == -1) {
+                op_errno = -ret;
+                goto err;
+        }
+
+        uuid_copy (local->parent_loc.gfid, local->parent_loc.inode->gfid);
+        GF_UUID_ASSERT (local->parent_loc.gfid);
+
+        STACK_WIND (frame, mq_release_parent_lock,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr,
+                    &local->parent_loc, newdict, 0, NULL);
+
+        ret = 0;
+err:
+        if (op_ret == -1 || ret == -1) {
+                local->err = op_errno;
+
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (newdict)
+                dict_unref (newdict);
+
+        return 0;
+}
+
+
+int32_t
+mq_update_parent_size (call_frame_t *frame,
+                       void *cookie,
+                       xlator_t *this,
+                       int32_t op_ret,
+                       int32_t op_errno,
+                       dict_t *dict, dict_t *xdata)
+{
+        int64_t             *size       = NULL;
+        int32_t              ret        = -1;
+        dict_t              *newdict    = NULL;
+        quota_local_t       *local      = NULL;
+        quota_inode_ctx_t   *ctx        = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, ((op_errno == ENOENT) ? GF_LOG_DEBUG :
+                                     GF_LOG_WARNING),
+                        "xattrop call failed: %s", strerror (op_errno));
+
+                goto err;
+        }
+
+        LOCK (&local->contri->lock);
+        {
+                local->contri->contribution += local->delta;
+        }
+        UNLOCK (&local->contri->lock);
+
+        gf_log (this->name, GF_LOG_DEBUG, "%s %"PRId64 "%"PRId64,
+                local->loc.path, local->ctx->size,
+                local->contri->contribution);
+
+        if (dict == NULL) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
+        if (ret < 0) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        newdict = dict_new ();
+        if (!newdict) {
+                op_errno = ENOMEM;
+                ret = -1;
+                goto err;
+        }
+
+        QUOTA_ALLOC_OR_GOTO (size, int64_t, ret, err);
+
+        *size = hton64 (local->delta);
+
+        ret = dict_set_bin (newdict, QUOTA_SIZE_KEY, size, 8);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto err;
+        }
+
+        if (uuid_is_null (local->parent_loc.gfid))
+                        uuid_copy (local->parent_loc.gfid,
+                                   local->parent_loc.inode->gfid);
+        GF_UUID_ASSERT (local->parent_loc.gfid);
+
+        STACK_WIND (frame,
+                    mq_mark_undirty,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop,
+                    &local->parent_loc,
+                    GF_XATTROP_ADD_ARRAY64,
+                    newdict, NULL);
+        ret = 0;
+err:
+        if (op_ret == -1 || ret < 0) {
+                local->err = op_errno;
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (newdict)
+                dict_unref (newdict);
+
+        return 0;
+}
+
+int32_t
+mq_update_inode_contribution (call_frame_t *frame, void *cookie,
+                              xlator_t *this, int32_t op_ret,
+                              int32_t op_errno, inode_t *inode,
+                              struct iatt *buf, dict_t *dict,
+                              struct iatt *postparent)
+{
+        int32_t               ret              = -1;
+        int64_t              *size             = NULL, size_int = 0, contri_int = 0;
+        int64_t              *contri           = NULL;
+        int64_t              *delta            = NULL;
+        char                  contri_key [512] = {0, };
+        dict_t               *newdict          = NULL;
+        quota_local_t        *local            = NULL;
+        quota_inode_ctx_t    *ctx              = NULL;
+        inode_contribution_t *contribution     = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, ((op_errno == ENOENT) ? GF_LOG_DEBUG :
+                                     GF_LOG_WARNING),
+                        "failed to get size and contribution of path (%s)(%s)",
+                        local->loc.path, strerror (op_errno));
+                goto err;
+        }
+
+        ctx = local->ctx;
+        contribution = local->contri;
+
+        //prepare to update size & contribution of the inode
+        GET_CONTRI_KEY (contri_key, contribution->gfid, ret);
+        if (ret == -1) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                if (local->loc.inode->ia_type == IA_IFDIR ) {
+                        ret = dict_get_bin (dict, QUOTA_SIZE_KEY,
+                                            (void **) &size);
+                        if (ret < 0) {
+                                op_errno = EINVAL;
+                                goto unlock;
+                        }
+
+                        ctx->size = ntoh64 (*size);
+                } else
+                        ctx->size = buf->ia_blocks * 512;
+
+                size_int = ctx->size;
+        }
+unlock:
+        UNLOCK  (&ctx->lock);
+
+        if (ret < 0) {
+                goto err;
+        }
+
+        ret = dict_get_bin (dict, contri_key, (void **) &contri);
+
+        LOCK (&contribution->lock);
+        {
+                if (ret < 0)
+                        contribution->contribution = 0;
+                else
+                        contribution->contribution = ntoh64 (*contri);
+
+                contri_int = contribution->contribution;
+        }
+        UNLOCK (&contribution->lock);
+
+        gf_log (this->name, GF_LOG_DEBUG, "%s %"PRId64 "%"PRId64,
+                local->loc.path, size_int, contri_int);
+
+        local->delta = size_int - contri_int;
+
+        if (local->delta == 0) {
+                mq_mark_undirty (frame, NULL, this, 0, 0, NULL, NULL);
+                return 0;
+        }
+
+        newdict = dict_new ();
+        if (newdict == NULL) {
+                op_errno = ENOMEM;
+                ret = -1;
+                goto err;
+        }
+
+        QUOTA_ALLOC_OR_GOTO (delta, int64_t, ret, err);
+
+        *delta = hton64 (local->delta);
+
+        ret = dict_set_bin (newdict, contri_key, delta, 8);
+        if (ret < 0) {
+                op_errno = -ret;
+                ret = -1;
+                goto err;
+        }
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame,
+                    mq_update_parent_size,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop,
+                    &local->loc,
+                    GF_XATTROP_ADD_ARRAY64,
+                    newdict, NULL);
+        ret = 0;
+
+err:
+        if (op_ret == -1 || ret < 0) {
+                local->err = op_errno;
+
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (newdict)
+                dict_unref (newdict);
+
+        return 0;
+}
+
+int32_t
+mq_fetch_child_size_and_contri (call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
+{
+        int32_t            ret              = -1;
+        char               contri_key [512] = {0, };
+        dict_t            *newdict          = NULL;
+        quota_local_t     *local            = NULL;
+        quota_inode_ctx_t *ctx              = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, (op_errno == ENOENT) ? GF_LOG_DEBUG
+                        : GF_LOG_WARNING,
+                        "couldnt mark inode corresponding to path (%s) dirty "
+                        "(%s)", local->parent_loc.path, strerror (op_errno));
+                goto err;
+        }
+
+        VALIDATE_OR_GOTO (local->ctx, err);
+        VALIDATE_OR_GOTO (local->contri, err);
+
+        gf_log (this->name, GF_LOG_DEBUG, "%s marked dirty", local->parent_loc.path);
+
+        //update parent ctx
+        ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
+        if (ret == -1) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->dirty = 1;
+        }
+        UNLOCK (&ctx->lock);
+
+        newdict = dict_new ();
+        if (newdict == NULL) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        if (local->loc.inode->ia_type == IA_IFDIR) {
+                ret = dict_set_int64 (newdict, QUOTA_SIZE_KEY, 0);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "dict_set failed.");
+                        goto err;
+                }
+        }
+
+        GET_CONTRI_KEY (contri_key, local->contri->gfid, ret);
+        if (ret < 0) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        ret = dict_set_int64 (newdict, contri_key, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict_set failed.");
+                goto err;
+        }
+
+        mq_set_ctx_updation_status (local->ctx, _gf_false);
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame, mq_update_inode_contribution, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &local->loc, newdict);
+
+        ret = 0;
+
+err:
+        if ((op_ret == -1) || (ret < 0)) {
+                local->err = op_errno;
+
+                mq_set_ctx_updation_status (local->ctx, _gf_false);
+
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (newdict)
+                dict_unref (newdict);
+
+        return 0;
+}
+
+int32_t
+mq_markdirty (call_frame_t *frame, void *cookie,
+              xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        int32_t        ret      = -1;
+        dict_t        *dict     = NULL;
+        quota_local_t *local    = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1){
+                gf_log (this->name, (op_errno == ENOENT) ? GF_LOG_DEBUG
+                        : GF_LOG_WARNING, "acquiring locks failed on %s (%s)",
+                        local->parent_loc.path, strerror (op_errno));
+
+                local->err = op_errno;
+
+                mq_set_ctx_updation_status (local->ctx, _gf_false);
+
+                mq_inodelk_cbk (frame, NULL, this, 0, 0, NULL);
+
+                return 0;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "inodelk succeeded on  %s", local->parent_loc.path);
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto err;
+        }
+
+        ret = dict_set_int8 (dict, QUOTA_DIRTY_KEY, 1);
+        if (ret == -1)
+                goto err;
+
+        uuid_copy (local->parent_loc.gfid,
+                   local->parent_loc.inode->gfid);
+        GF_UUID_ASSERT (local->parent_loc.gfid);
+
+        STACK_WIND (frame, mq_fetch_child_size_and_contri,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr,
+                    &local->parent_loc, dict, 0, NULL);
+
+        ret = 0;
+err:
+        if (ret == -1) {
+                local->err = 1;
+
+                mq_set_ctx_updation_status (local->ctx, _gf_false);
+
+                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (dict)
+                dict_unref (dict);
+
+        return 0;
+}
+
+
+int32_t
+mq_get_lock_on_parent (call_frame_t *frame, xlator_t *this)
+{
+        struct gf_flock  lock  = {0, };
+        quota_local_t   *local = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", frame, fr_destroy);
+
+        local = frame->local;
+        gf_log (this->name, GF_LOG_DEBUG, "taking lock on %s",
+                local->parent_loc.path);
+
+        if (local->parent_loc.inode == NULL) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "parent inode is not valid, aborting "
+                        "transaction.");
+                goto fr_destroy;
+        }
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND (frame,
+                    mq_markdirty,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
+
+        return 0;
+
+fr_destroy:
+        QUOTA_STACK_DESTROY (frame, this);
+
+        return -1;
+}
+
+
+int
+mq_start_quota_txn (xlator_t *this, loc_t *loc,
+                    quota_inode_ctx_t *ctx,
+                    inode_contribution_t *contri)
+{
+        int32_t        ret      = -1;
+        call_frame_t  *frame    = NULL;
+        quota_local_t *local    = NULL;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (frame == NULL)
+                goto err;
+
+        mq_assign_lk_owner (this, frame);
+
+        local = mq_local_new ();
+        if (local == NULL)
+                goto fr_destroy;
+
+        frame->local = local;
+
+        ret = mq_loc_copy (&local->loc, loc);
+        if (ret < 0)
+                goto fr_destroy;
+
+        ret = mq_inode_loc_fill (NULL, local->loc.parent,
+                                 &local->parent_loc);
+        if (ret < 0)
+                goto fr_destroy;
+
+        local->ctx = ctx;
+        local->contri = contri;
+
+        ret = mq_get_lock_on_parent (frame, this);
+        if (ret == -1)
+                goto err;
+
+        return 0;
+
+fr_destroy:
+        QUOTA_STACK_DESTROY (frame, this);
+err:
+        mq_set_ctx_updation_status (ctx, _gf_false);
+
+        return -1;
+}
+
+
+int
+mq_initiate_quota_txn (xlator_t *this, loc_t *loc)
+{
+        int32_t               ret          = -1;
+        gf_boolean_t          status       = _gf_false;
+        quota_inode_ctx_t    *ctx          = NULL;
+        inode_contribution_t *contribution = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", loc, out);
+        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+
+        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "inode ctx get failed, aborting quota txn");
+                ret = -1;
+                goto out;
+        }
+
+        contribution = mq_get_contribution_node (loc->parent, ctx);
+        if (contribution == NULL)
+                goto out;
+
+        /* To improve performance, donot start another transaction
+         * if one is already in progress for same inode
+         */
+        status = _gf_true;
+
+        ret = mq_test_and_set_ctx_updation_status (ctx, &status);
+        if (ret < 0)
+                goto out;
+
+        if (status == _gf_false) {
+                mq_start_quota_txn (this, loc, ctx, contribution);
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+/* int32_t */
+/* validate_inode_size_contribution (xlator_t *this, loc_t *loc, int64_t size, */
+/*                                int64_t contribution) */
+/* { */
+/*   if (size != contribution) { */
+/*     mq_initiate_quota_txn (this, loc); */
+/*   } */
+
+/*   return 0; */
+/* } */
+
+
+int32_t
+mq_inspect_directory_xattr (xlator_t *this,
+                            loc_t *loc,
+                            dict_t *dict,
+                            struct iatt buf)
+{
+        int32_t               ret                 = 0;
+        int8_t                dirty               = -1;
+        int64_t              *size                = NULL, size_int = 0;
+        int64_t              *contri              = NULL, contri_int = 0;
+        char                  contri_key [512]    = {0, };
+        gf_boolean_t          not_root            = _gf_false;
+        quota_inode_ctx_t    *ctx                 = NULL;
+        inode_contribution_t *contribution        = NULL;
+
+        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
+        if (ret < 0) {
+                ctx = mq_inode_ctx_new (loc->inode, this);
+                if (ctx == NULL) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "mq_inode_ctx_new failed");
+                        ret = -1;
+                        goto err;
+                }
+        }
+
+        if (strcmp (loc->path, "/") != 0) {
+                contribution = mq_add_new_contribution_node (this, ctx, loc);
+                if (contribution == NULL) {
+                        if (!uuid_is_null (loc->inode->gfid))
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "cannot add a new contribution node");
+                        ret = -1;
+                        goto err;
+                }
+        }
+
+        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+        if (ret < 0)
+                goto out;
+
+        ret = dict_get_int8 (dict, QUOTA_DIRTY_KEY, &dirty);
+        if (ret < 0)
+                goto out;
+
+        if (strcmp (loc->path, "/") != 0) {
+                not_root = _gf_true;
+
+                GET_CONTRI_KEY (contri_key, contribution->gfid, ret);
+                if (ret < 0)
+                        goto out;
+
+                ret = dict_get_bin (dict, contri_key, (void **) &contri);
+                if (ret < 0)
+                        goto out;
+
+                LOCK (&contribution->lock);
+                {
+                        contribution->contribution = ntoh64 (*contri);
+                        contri_int = contribution->contribution;
+                }
+                UNLOCK (&contribution->lock);
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->size = ntoh64 (*size);
+                ctx->dirty = dirty;
+                size_int = ctx->size;
+        }
+        UNLOCK (&ctx->lock);
+
+        gf_log (this->name, GF_LOG_DEBUG, "size=%"PRId64
+                " contri=%"PRId64, size_int, contri_int);
+
+        if (dirty) {
+                ret = mq_update_dirty_inode (this, loc, ctx, contribution);
+        }
+
+        if ((!dirty || ret == 0) && (not_root == _gf_true) &&
+            (size_int != contri_int)) {
+                mq_initiate_quota_txn (this, loc);
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                mq_set_inode_xattr (this, loc);
+err:
+        return ret;
+}
+
+int32_t
+mq_inspect_file_xattr (xlator_t *this,
+                       loc_t *loc,
+                       dict_t *dict,
+                       struct iatt buf)
+{
+        int32_t               ret              = -1;
+        uint64_t              contri_int       = 0, size = 0;
+        int64_t              *contri_ptr       = NULL;
+        char                  contri_key [512] = {0, };
+        quota_inode_ctx_t    *ctx              = NULL;
+        inode_contribution_t *contribution     = NULL;
+
+        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
+        if (ret < 0) {
+                ctx = mq_inode_ctx_new (loc->inode, this);
+                if (ctx == NULL) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "mq_inode_ctx_new failed");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        contribution = mq_add_new_contribution_node (this, ctx, loc);
+        if (contribution == NULL)
+                goto out;
+
+        LOCK (&ctx->lock);
+        {
+                ctx->size = 512 * buf.ia_blocks;
+                size = ctx->size;
+        }
+        UNLOCK (&ctx->lock);
+
+        list_for_each_entry (contribution, &ctx->contribution_head,
+                             contri_list) {
+                GET_CONTRI_KEY (contri_key, contribution->gfid, ret);
+                if (ret < 0)
+                        continue;
+
+                ret = dict_get_bin (dict, contri_key, (void **) &contri_int);
+                if (ret == 0) {
+                        contri_ptr = (int64_t *)(unsigned long)contri_int;
+
+                        LOCK (&contribution->lock);
+                        {
+                                contribution->contribution = ntoh64 (*contri_ptr);
+                                contri_int = contribution->contribution;
+                        }
+                        UNLOCK (&contribution->lock);
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "size=%"PRId64 " contri=%"PRId64, size, contri_int);
+
+                        if (size != contri_int) {
+                                mq_initiate_quota_txn (this, loc);
+                        }
+                } else
+                        mq_initiate_quota_txn (this, loc);
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+mq_xattr_state (xlator_t *this,
+                loc_t *loc,
+                dict_t *dict,
+                struct iatt buf)
+{
+        if (buf.ia_type == IA_IFREG ||
+            buf.ia_type == IA_IFLNK) {
+                mq_inspect_file_xattr (this, loc, dict, buf);
+        } else if (buf.ia_type == IA_IFDIR)
+                mq_inspect_directory_xattr (this, loc, dict, buf);
+
+        return 0;
+}
+
+int32_t
+mq_req_xattr (xlator_t *this,
+              loc_t *loc,
+              dict_t *dict)
+{
+        int32_t               ret       = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", dict, out);
+
+        if (!loc)
+                goto set_size;
+
+        //if not "/" then request contribution
+        if (strcmp (loc->path, "/") == 0)
+                goto set_size;
+
+        ret = mq_dict_set_contribution (this, dict, loc);
+        if (ret == -1)
+                goto out;
+
+set_size:
+        ret = dict_set_uint64 (dict, QUOTA_SIZE_KEY, 0);
+        if (ret < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_int8 (dict, QUOTA_DIRTY_KEY, 0);
+        if (ret < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+int32_t
+mq_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        QUOTA_STACK_DESTROY (frame, this);
+
+        return 0;
+}
+
+int32_t
+_mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        int32_t        ret                = 0;
+        char           contri_key [512]   = {0, };
+        quota_local_t *local              = NULL;
+
+        local = (quota_local_t *) frame->local;
+
+        if (op_ret == -1 || local->err == -1) {
+                mq_removexattr_cbk (frame, NULL, this, -1, 0, NULL);
+                return 0;
+        }
+
+        frame->local = NULL;
+
+        if (local->hl_count > 1) {
+                GET_CONTRI_KEY (contri_key, local->contri->gfid, ret);
+
+                STACK_WIND (frame, mq_removexattr_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->removexattr,
+                            &local->loc, contri_key, NULL);
+                ret = 0;
+        } else {
+                mq_removexattr_cbk (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (strcmp (local->parent_loc.path, "/") != 0) {
+                ret = mq_get_parent_inode_local (this, local);
+                if (ret < 0)
+                        goto out;
+
+                mq_start_quota_txn (this, &local->loc, local->ctx, local->contri);
+        }
+out:
+        mq_local_unref (this, local);
+
+        return 0;
+}
+
+int32_t
+mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
+{
+        int32_t            ret   = -1;
+        struct gf_flock    lock  = {0, };
+        quota_inode_ctx_t *ctx   = NULL;
+        quota_local_t     *local = NULL;
+        int64_t contribution = 0;
+
+        local = frame->local;
+        if (op_ret == -1)
+                local->err = -1;
+
+        ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
+
+        LOCK (&local->contri->lock);
+        {
+                contribution = local->contri->contribution;
+        }
+        UNLOCK (&local->contri->lock);
+
+        if (contribution == local->size) {
+                if (ret == 0) {
+                        LOCK (&ctx->lock);
+                        {
+                                ctx->size -= contribution;
+                        }
+                        UNLOCK (&ctx->lock);
+
+                        LOCK (&local->contri->lock);
+                        {
+                                local->contri->contribution = 0;
+                        }
+                        UNLOCK (&local->contri->lock);
+                }
+        }
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+        STACK_WIND (frame,
+                    _mq_inode_remove_done,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->parent_loc,
+                    F_SETLKW, &lock, NULL);
+        return 0;
+}
+
+int32_t
+mq_reduce_parent_size_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        int32_t                  ret               = -1;
+        int64_t                 *size              = NULL;
+        dict_t                  *dict              = NULL;
+        quota_local_t           *local             = NULL;
+
+        local = frame->local;
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "inodelk set failed on %s", local->parent_loc.path);
+                QUOTA_STACK_DESTROY (frame, this);
+                return 0;
+        }
+
+        VALIDATE_OR_GOTO (local->contri, err);
+
+        dict = dict_new ();
+        if (dict == NULL) {
+                ret = -1;
+                goto err;
+        }
+
+        QUOTA_ALLOC_OR_GOTO (size, int64_t, ret, err);
+
+        *size = hton64 (-local->size);
+
+        ret = dict_set_bin (dict, QUOTA_SIZE_KEY, size, 8);
+        if (ret < 0)
+                goto err;
+
+        uuid_copy (local->parent_loc.gfid,
+                   local->parent_loc.inode->gfid);
+        GF_UUID_ASSERT (local->parent_loc.gfid);
+
+        STACK_WIND (frame, mq_inode_remove_done, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop, &local->parent_loc,
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
+        dict_unref (dict);
+        return 0;
+
+err:
+        local->err = 1;
+        mq_inode_remove_done (frame, NULL, this, -1, 0, NULL, NULL);
+        if (dict)
+                dict_unref (dict);
+        return 0;
+}
+
+int32_t
+mq_reduce_parent_size (xlator_t *this, loc_t *loc, int64_t contri)
+{
+        int32_t                  ret           = -1;
+        struct gf_flock          lock          = {0,};
+        call_frame_t            *frame         = NULL;
+        quota_local_t           *local         = NULL;
+        quota_inode_ctx_t       *ctx           = NULL;
+        inode_contribution_t    *contribution  = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", loc, out);
+
+        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
+        if (ret < 0)
+                goto out;
+
+        contribution = mq_get_contribution_node (loc->parent, ctx);
+        if (contribution == NULL)
+                goto out;
+
+        local = mq_local_new ();
+        if (local == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        if (contri >= 0) {
+                local->size = contri;
+        } else {
+                LOCK (&contribution->lock);
+                {
+                        local->size = contribution->contribution;
+                }
+                UNLOCK (&contribution->lock);
+        }
+
+        if (local->size == 0) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = mq_loc_copy (&local->loc, loc);
+        if (ret < 0)
+                goto out;
+
+        local->ctx = ctx;
+        local->contri = contribution;
+
+        ret = mq_inode_loc_fill (NULL, loc->parent, &local->parent_loc);
+        if (ret < 0)
+                goto out;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto out;
+        }
+
+        mq_assign_lk_owner (this, frame);
+
+        frame->local = local;
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        if (local->parent_loc.inode == NULL) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Inode is NULL, so can't stackwind.");
+                goto out;
+        }
+
+        STACK_WIND (frame,
+                    mq_reduce_parent_size_xattr,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
+        local = NULL;
+        ret = 0;
+
+out:
+        if (local != NULL)
+                mq_local_unref (this, local);
+
+        return ret;
+}
+
+
+int32_t
+init_quota_priv (xlator_t *this)
+{
+        return 0;
+}
+
+
+int32_t
+mq_rename_update_newpath (xlator_t *this, loc_t *loc)
+{
+        int32_t               ret          = -1;
+        quota_inode_ctx_t    *ctx          = NULL;
+        inode_contribution_t *contribution = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", loc, out);
+        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+
+        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
+        if (ret < 0)
+                goto out;
+
+        contribution = mq_add_new_contribution_node (this, ctx, loc);
+        if (contribution == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        mq_initiate_quota_txn (this, loc);
+out:
+        return ret;
+}
+
+int32_t
+mq_forget (xlator_t *this, quota_inode_ctx_t *ctx)
+{
+        inode_contribution_t *contri = NULL;
+        inode_contribution_t *next   = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+
+        list_for_each_entry_safe (contri, next, &ctx->contribution_head,
+                                  contri_list) {
+                list_del (&contri->contri_list);
+                GF_FREE (contri);
+        }
+
+        LOCK_DESTROY (&ctx->lock);
+        GF_FREE (ctx);
+out:
+        return 0;
+}
diff --git a/xlators/features/marker/src/marker-quota.h b/xlators/features/marker/src/marker-quota.h
new file mode 100644
index 000000000..385760ac4
--- /dev/null
+++ b/xlators/features/marker/src/marker-quota.h
@@ -0,0 +1,130 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _MARKER_QUOTA_H
+#define _MARKER_QUOTA_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "marker-mem-types.h"
+
+#define QUOTA_XATTR_PREFIX "trusted.glusterfs"
+#define QUOTA_DIRTY_KEY "trusted.glusterfs.quota.dirty"
+
+#define CONTRIBUTION "contri"
+#define CONTRI_KEY_MAX 512
+#define READDIR_BUF 4096
+
+
+#define QUOTA_STACK_DESTROY(_frame, _this)              \
+        do {                                            \
+                quota_local_t *_local = NULL;           \
+                _local = _frame->local;                 \
+                _frame->local = NULL;                   \
+                STACK_DESTROY (_frame->root);           \
+                mq_local_unref (_this, _local);         \
+        } while (0)
+
+
+#define QUOTA_ALLOC(var, type, ret)                     \
+        do {                                            \
+                ret = 0;                                \
+                var = GF_CALLOC (sizeof (type), 1,      \
+                                 gf_marker_mt_##type);  \
+                if (!var) {                             \
+                        gf_log ("", GF_LOG_ERROR,       \
+                                "out of memory");       \
+                        ret = -1;                       \
+                }                                       \
+        } while (0);
+
+#define QUOTA_ALLOC_OR_GOTO(var, type, ret, label)      \
+        do {                                            \
+                var = GF_CALLOC (sizeof (type), 1,      \
+                                 gf_marker_mt_##type);  \
+                if (!var) {                             \
+                        gf_log ("", GF_LOG_ERROR,       \
+                                "out of memory");       \
+                        ret = -1;                       \
+                        goto label;                     \
+                }                                       \
+                ret = 0;                                \
+        } while (0);
+
+#define GET_CONTRI_KEY(var, _gfid, _ret)        \
+        do {                                    \
+                char _gfid_unparsed[40];        \
+                uuid_unparse (_gfid, _gfid_unparsed); \
+                _ret = snprintf (var, CONTRI_KEY_MAX, QUOTA_XATTR_PREFIX \
+                                 ".%s.%s." CONTRIBUTION, "quota", \
+                                 _gfid_unparsed); \
+        } while (0);
+
+#define QUOTA_SAFE_INCREMENT(lock, var)         \
+        do {                                    \
+                LOCK (lock);                    \
+                var ++;                         \
+                UNLOCK (lock);                  \
+        } while (0)
+
+struct quota_inode_ctx {
+        int64_t                size;
+        int8_t                 dirty;
+        gf_boolean_t           updation_status;
+        gf_lock_t              lock;
+        struct list_head       contribution_head;
+};
+typedef struct quota_inode_ctx quota_inode_ctx_t;
+
+struct inode_contribution {
+        struct list_head contri_list;
+        int64_t          contribution;
+        uuid_t           gfid;
+  gf_lock_t lock;
+};
+typedef struct inode_contribution inode_contribution_t;
+
+int32_t
+mq_get_lock_on_parent (call_frame_t *, xlator_t *);
+
+int32_t
+mq_req_xattr (xlator_t *, loc_t *, dict_t *);
+
+int32_t
+init_quota_priv (xlator_t *);
+
+int32_t
+mq_xattr_state (xlator_t *, loc_t *, dict_t *, struct iatt);
+
+int32_t
+mq_set_inode_xattr (xlator_t *, loc_t *);
+
+int
+mq_initiate_quota_txn (xlator_t *, loc_t *);
+
+int32_t
+mq_dirty_inode_readdir (call_frame_t *, void *, xlator_t *,
+                        int32_t, int32_t, fd_t *, dict_t *);
+
+int32_t
+mq_reduce_parent_size (xlator_t *, loc_t *, int64_t);
+
+int32_t
+mq_rename_update_newpath (xlator_t *, loc_t *);
+
+int32_t
+mq_inspect_file_xattr (xlator_t *this, loc_t *loc, dict_t *dict, struct iatt buf);
+
+int32_t
+mq_forget (xlator_t *, quota_inode_ctx_t *);
+#endif
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
new file mode 100644
index 000000000..6a2c85691
--- /dev/null
+++ b/xlators/features/marker/src/marker.c
@@ -0,0 +1,2862 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "libxlator.h"
+#include "marker.h"
+#include "marker-mem-types.h"
+#include "marker-quota.h"
+#include "marker-quota-helper.h"
+#include "marker-common.h"
+#include "byte-order.h"
+
+#define _GF_UID_GID_CHANGED 1
+
+void
+fini (xlator_t *this);
+
+int32_t
+marker_start_setxattr (call_frame_t *, xlator_t *);
+
+marker_local_t *
+marker_local_ref (marker_local_t *local)
+{
+        GF_VALIDATE_OR_GOTO ("marker", local, err);
+
+        LOCK (&local->lock);
+        {
+                local->ref++;
+        }
+        UNLOCK (&local->lock);
+
+        return local;
+err:
+        return NULL;
+}
+
+int
+marker_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+{
+        int     ret = -1;
+
+        if (!loc)
+                return ret;
+
+        if (inode) {
+                loc->inode = inode_ref (inode);
+                if (uuid_is_null (loc->gfid)) {
+                        uuid_copy (loc->gfid, loc->inode->gfid);
+                }
+        }
+
+        if (parent)
+                loc->parent = inode_ref (parent);
+
+        if (path) {
+                loc->path = gf_strdup (path);
+                if (!loc->path) {
+                        gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
+                        goto loc_wipe;
+                }
+
+                loc->name = strrchr (loc->path, '/');
+                if (loc->name)
+                        loc->name++;
+        }
+
+        ret = 0;
+loc_wipe:
+        if (ret < 0)
+                loc_wipe (loc);
+
+        return ret;
+}
+
+int
+marker_inode_loc_fill (inode_t *inode, loc_t *loc)
+{
+        char            *resolvedpath = NULL;
+        int              ret          = -1;
+	inode_t         *parent = NULL;
+
+        if ((!inode) || (!loc))
+                return ret;
+
+	parent = inode_parent (inode, NULL, NULL);
+
+        ret = inode_path (inode, NULL, &resolvedpath);
+        if (ret < 0)
+                goto err;
+
+        ret = marker_loc_fill (loc, inode, parent, resolvedpath);
+        if (ret < 0)
+                goto err;
+
+err:
+	if (parent)
+		inode_unref (parent);
+
+        GF_FREE (resolvedpath);
+
+        return ret;
+}
+
+int32_t
+marker_trav_parent (marker_local_t *local)
+{
+        int32_t ret = 0;
+        loc_t   loc = {0, };
+        inode_t *parent = NULL;
+        int8_t  need_unref = 0;
+
+        if (!local->loc.parent) {
+                parent = inode_parent (local->loc.inode, NULL, NULL);
+                if (parent)
+                        need_unref = 1;
+        } else
+                parent = local->loc.parent;
+
+        ret = marker_inode_loc_fill (parent, &loc);
+
+        if (ret < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        loc_wipe (&local->loc);
+
+        local->loc = loc;
+out:
+        if (need_unref)
+                inode_unref (parent);
+
+        return ret;
+}
+
+int32_t
+marker_error_handler (xlator_t *this, marker_local_t *local, int32_t op_errno)
+{
+         marker_conf_t *priv = NULL;
+         const char    *path = NULL;
+
+         priv = (marker_conf_t *) this->private;
+         path = local
+                 ? (local->loc.path
+                    ? local->loc.path : uuid_utoa(local->loc.gfid))
+                 : "<nul>";
+
+         gf_log (this->name, GF_LOG_CRITICAL,
+                 "Indexing gone corrupt at %s (reason: %s)."
+                 " Geo-replication slave content needs to be revalidated",
+                 path, strerror (op_errno));
+        unlink (priv->timestamp_file);
+
+        return 0;
+}
+
+int32_t
+marker_local_unref (marker_local_t *local)
+{
+        int32_t   var = 0;
+
+        if (local == NULL)
+                return -1;
+
+        LOCK (&local->lock);
+        {
+                var = --local->ref;
+        }
+        UNLOCK (&local->lock);
+
+        if (var != 0)
+                goto out;
+
+        loc_wipe (&local->loc);
+        loc_wipe (&local->parent_loc);
+        if (local->xdata)
+                dict_unref (local->xdata);
+
+        if (local->oplocal) {
+                marker_local_unref (local->oplocal);
+                local->oplocal = NULL;
+        }
+        mem_put (local);
+out:
+        return 0;
+}
+
+int32_t
+stat_stampfile (xlator_t *this, marker_conf_t *priv,
+                struct volume_mark **status)
+{
+        struct stat          buf      = {0, };
+        struct volume_mark  *vol_mark = NULL;
+
+        vol_mark = GF_CALLOC (sizeof (struct volume_mark), 1,
+                              gf_marker_mt_volume_mark);
+
+        vol_mark->major = 1;
+        vol_mark->minor = 0;
+
+        GF_ASSERT (sizeof (priv->volume_uuid_bin) == 16);
+        memcpy (vol_mark->uuid, priv->volume_uuid_bin, 16);
+
+        if (stat (priv->timestamp_file, &buf) != -1) {
+                vol_mark->retval = 0;
+                vol_mark->sec = htonl (buf.st_ctime);
+                vol_mark->usec = htonl (ST_CTIM_NSEC (&buf)/1000);
+        } else
+                vol_mark->retval = 1;
+
+        *status = vol_mark;
+
+        return 0;
+}
+
+int32_t
+marker_getxattr_stampfile_cbk (call_frame_t *frame, xlator_t *this,
+                               const char *name, struct volume_mark *vol_mark,
+                               dict_t *xdata)
+{
+        int32_t   ret  = -1;
+        dict_t   *dict = NULL;
+
+        if (vol_mark == NULL){
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
+
+                goto out;
+        }
+
+        dict = dict_new ();
+
+        ret = dict_set_bin (dict, (char *)name, vol_mark,
+                            sizeof (struct volume_mark));
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "failed to set key %s",
+                        name);
+
+        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, dict, xdata);
+
+        dict_unref (dict);
+out:
+        return 0;
+}
+
+int32_t
+call_from_special_client (call_frame_t *frame, xlator_t *this, const char *name)
+{
+        struct volume_mark     *vol_mark   = NULL;
+        marker_conf_t          *priv       = NULL;
+        gf_boolean_t            ret        = _gf_true;
+
+        priv = (marker_conf_t *)this->private;
+
+        if (frame->root->pid != GF_CLIENT_PID_GSYNCD || name == NULL ||
+            strcmp (name, MARKER_XATTR_PREFIX "." VOLUME_MARK) != 0) {
+                ret = _gf_false;
+                goto out;
+        }
+
+        stat_stampfile (this, priv, &vol_mark);
+
+        marker_getxattr_stampfile_cbk (frame, this, name, vol_mark, NULL);
+out:
+        return ret;
+}
+
+int32_t
+marker_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        if (cookie) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Filtering the quota extended attributes");
+
+                dict_foreach_fnmatch (dict, "trusted.glusterfs.quota*",
+                                      marker_filter_quota_xattr, NULL);
+        }
+
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+int32_t
+marker_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
+{
+        gf_boolean_t   ret    = _gf_false;
+        marker_conf_t *priv   = NULL;
+        unsigned long  cookie = 0;
+
+        priv = this->private;
+
+        if (priv == NULL || (priv->feature_enabled & GF_XTIME) == 0)
+                goto wind;
+
+        gf_log (this->name, GF_LOG_DEBUG, "USER:PID = %d", frame->root->pid);
+
+        ret = call_from_special_client (frame, this, name);
+wind:
+        if (ret == _gf_false) {
+                if (name == NULL) {
+                        /* Signifies that marker translator
+                         * has to filter the quota's xattr's,
+                         * this is to prevent afr from performing
+                         * self healing on marker-quota xattrs'
+                         */
+                        cookie = 1;
+                }
+                STACK_WIND_COOKIE (frame, marker_getxattr_cbk, (void *)cookie,
+                                   FIRST_CHILD(this),
+                                   FIRST_CHILD(this)->fops->getxattr, loc,
+                                   name, xdata);
+        }
+
+        return 0;
+}
+
+
+int32_t
+marker_setxattr_done (call_frame_t *frame)
+{
+        marker_local_t *local = NULL;
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_DESTROY (frame->root);
+
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int
+marker_specific_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        int32_t         ret   = 0;
+        int32_t         done  = 0;
+        marker_local_t *local = NULL;
+
+        local = (marker_local_t*) frame->local;
+
+        if (op_ret == -1 && op_errno == ENOSPC) {
+                marker_error_handler (this, local, op_errno);
+                done = 1;
+                goto out;
+        }
+
+        if (local) {
+                if (local->loc.path && strcmp (local->loc.path, "/") == 0) {
+                        done = 1;
+                        goto out;
+                }
+                if (__is_root_gfid (local->loc.gfid)) {
+                        done = 1;
+                        goto out;
+                }
+        }
+
+        ret = marker_trav_parent (local);
+
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "Error occurred "
+                        "while traversing to the parent, stopping marker");
+
+                done = 1;
+
+                goto out;
+        }
+
+        marker_start_setxattr (frame, this);
+
+out:
+        if (done) {
+                marker_setxattr_done (frame);
+        }
+
+        return 0;
+}
+
+int32_t
+marker_start_setxattr (call_frame_t *frame, xlator_t *this)
+{
+        int32_t          ret   = -1;
+        dict_t          *dict  = NULL;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        local = (marker_local_t*) frame->local;
+
+        if (!local)
+                goto out;
+
+        dict = dict_new ();
+
+        if (!dict)
+                goto out;
+
+        if (local->loc.inode && uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        ret = dict_set_static_bin (dict, priv->marker_xattr,
+                                   (void *)local->timebuf, 8);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set marker xattr (%s)", local->loc.path);
+                goto out;
+        }
+
+        STACK_WIND (frame, marker_specific_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, &local->loc, dict, 0,
+                    NULL);
+
+        ret = 0;
+out:
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+void
+marker_gettimeofday (marker_local_t *local)
+{
+        struct timeval tv = {0, };
+
+        gettimeofday (&tv, NULL);
+
+        local->timebuf [0] = htonl (tv.tv_sec);
+        local->timebuf [1] = htonl (tv.tv_usec);
+
+        return;
+}
+
+int32_t
+marker_create_frame (xlator_t *this, marker_local_t *local)
+{
+        call_frame_t *frame = NULL;
+
+        frame = create_frame (this, this->ctx->pool);
+
+        frame->local = (void *) local;
+
+        marker_start_setxattr (frame, this);
+
+        return 0;
+}
+
+int32_t
+marker_xtime_update_marks (xlator_t *this, marker_local_t *local)
+{
+        marker_conf_t *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+        priv = this->private;
+
+        if ((local->pid == GF_CLIENT_PID_GSYNCD
+             && !(priv->feature_enabled & GF_XTIME_GSYNC_FORCE))
+            || (local->pid == GF_CLIENT_PID_DEFRAG))
+                goto out;
+
+        marker_gettimeofday (local);
+
+        marker_local_ref (local);
+
+        marker_create_frame (this, local);
+out:
+        return 0;
+}
+
+
+int32_t
+marker_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        marker_conf_t      *priv    = NULL;
+        marker_local_t     *local   = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "error occurred "
+                        "while Creating a file %s", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode,
+                             buf, preparent, postparent, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_set_inode_xattr (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int
+marker_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              mode_t umask, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (mkdir, frame, -1, ENOMEM, NULL,
+                             NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "error occurred "
+                        "while Creating a file %s", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_set_inode_xattr (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_create_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                             NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        marker_conf_t      *priv    = NULL;
+        marker_local_t     *local   = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "error occurred "
+                        "while write, %s", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_writev (call_frame_t *frame,
+               xlator_t *this,
+               fd_t *fd,
+               struct iovec *vector,
+               int32_t count,
+               off_t offset, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_writev_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+                    flags, iobref, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        marker_conf_t      *priv    = NULL;
+        marker_local_t     *local   = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "error occurred "
+                        "rmdir %s", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_reduce_parent_size (this, &local->loc, -1);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_rmdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        marker_conf_t      *priv    = NULL;
+        marker_local_t     *local   = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s occurred in unlink", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if ((priv->feature_enabled & GF_QUOTA) && (local->ia_nlink == 1))
+                mq_reduce_parent_size (this, &local->loc, -1);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+
+int32_t
+marker_unlink_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        dict_t *xdata)
+{
+        marker_local_t *local = NULL;
+
+        local = frame->local;
+        if (op_ret < 0) {
+                goto err;
+        }
+
+        if (local == NULL) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->ia_nlink = buf->ia_nlink;
+
+        STACK_WIND (frame, marker_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, &local->loc, local->xflag,
+                    local->xdata);
+        return 0;
+err:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+        marker_local_unref (local);
+        return 0;
+}
+
+
+int32_t
+marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto unlink_wind;
+
+        local = mem_get0 (this->local_pool);
+        local->xflag = xflag;
+        if (xdata)
+                local->xdata = dict_ref (xdata);
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        if (ret == -1)
+                goto err;
+
+        if (uuid_is_null (loc->gfid) && loc->inode)
+                uuid_copy (loc->gfid, loc->inode->gfid);
+
+        STACK_WIND (frame, marker_unlink_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+
+unlink_wind:
+        STACK_WIND (frame, marker_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+err:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+        marker_local_unref (local);
+        return 0;
+}
+
+
+int32_t
+marker_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "linking a file ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, newloc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_link_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                             NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        marker_local_t *local  = NULL, *oplocal = NULL;
+        loc_t           newloc = {0, };
+        marker_conf_t  *priv   = NULL;
+
+        local = frame->local;
+        oplocal = local->oplocal;
+
+        priv = this->private;
+
+        frame->local = NULL;
+
+        if (op_ret < 0) {
+                if (local->err == 0) {
+                        local->err = op_errno;
+                }
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "inodelk (UNLOCK) failed on path:%s (gfid:%s) (%s)",
+                        local->parent_loc.path,
+                        uuid_utoa (local->parent_loc.inode->gfid),
+                        strerror (op_errno));
+        }
+
+        if (local->stub != NULL) {
+                call_resume (local->stub);
+                local->stub = NULL;
+        } else if (local->err != 0) {
+                STACK_UNWIND_STRICT (rename, frame, -1, local->err, NULL, NULL,
+                                     NULL, NULL, NULL, NULL);
+        }
+
+        mq_reduce_parent_size (this, &oplocal->loc, oplocal->contribution);
+
+        if (local->loc.inode != NULL) {
+                mq_reduce_parent_size (this, &local->loc, local->contribution);
+        }
+
+        newloc.inode = inode_ref (oplocal->loc.inode);
+        newloc.path = gf_strdup (local->loc.path);
+        newloc.name = strrchr (newloc.path, '/');
+        if (newloc.name)
+                newloc.name++;
+        newloc.parent = inode_ref (local->loc.parent);
+
+        mq_rename_update_newpath (this, &newloc);
+
+        loc_wipe (&newloc);
+
+        if (priv->feature_enabled & GF_XTIME) {
+                //update marks on oldpath
+                uuid_copy (local->loc.gfid, oplocal->loc.inode->gfid);
+                marker_xtime_update_marks (this, oplocal);
+                marker_xtime_update_marks (this, local);
+        }
+
+        marker_local_unref (local);
+        marker_local_unref (oplocal);
+        return 0;
+}
+
+
+int32_t
+marker_rename_release_newp_lock (call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *xdata)
+{
+        marker_local_t  *local = NULL, *oplocal = NULL;
+        struct gf_flock  lock  = {0, };
+
+        local = frame->local;
+        oplocal = local->oplocal;
+
+        if (op_ret < 0) {
+                if (local->err == 0) {
+                        local->err = op_errno;
+                }
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "inodelk (UNLOCK) failed on %s (gfid:%s) (%s)",
+                        oplocal->parent_loc.path,
+                        uuid_utoa (oplocal->parent_loc.inode->gfid),
+                        strerror (op_errno));
+        }
+
+        if (local->next_lock_on == NULL) {
+                marker_rename_done (frame, NULL, this, 0, 0, NULL);
+                goto out;
+        }
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+        STACK_WIND (frame,
+                    marker_rename_done,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
+
+out:
+        return 0;
+}
+
+
+int32_t
+marker_rename_release_oldp_lock (call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *xdata)
+{
+        marker_local_t  *local = NULL, *oplocal = NULL;
+        struct gf_flock  lock  = {0, };
+
+        local = frame->local;
+        oplocal = local->oplocal;
+
+        if ((op_ret < 0) && (op_errno != ENOATTR)) {
+                local->err = op_errno;
+        }
+
+        //Reset frame uid and gid if set.
+        if (cookie == (void *) _GF_UID_GID_CHANGED)
+                MARKER_RESET_UID_GID (frame, frame->root, local);
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+        STACK_WIND (frame,
+                    marker_rename_release_newp_lock,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &oplocal->parent_loc, F_SETLKW, &lock, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+        marker_conf_t  *priv                 = NULL;
+        marker_local_t *local                = NULL;
+        marker_local_t *oplocal              = NULL;
+        call_stub_t    *stub                 = NULL;
+        int32_t         ret                  = 0;
+        char            contri_key [512]     = {0, };
+        loc_t           newloc               = {0, };
+
+        local = (marker_local_t *) frame->local;
+
+        if (local != NULL) {
+                oplocal = local->oplocal;
+        }
+
+        priv = this->private;
+
+        if (op_ret < 0) {
+                if (local != NULL) {
+                        local->err = op_errno;
+                }
+
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "renaming a file ", strerror (op_errno));
+        }
+
+        if (priv->feature_enabled & GF_QUOTA) {
+                if ((op_ret < 0) || (local == NULL)) {
+                        goto quota_err;
+                }
+
+                stub = fop_rename_cbk_stub (frame, default_rename_cbk, op_ret,
+                                            op_errno, buf, preoldparent,
+                                            postoldparent, prenewparent,
+                                            postnewparent, xdata);
+                if (stub == NULL) {
+                        local->err = ENOMEM;
+                        goto quota_err;
+                }
+
+                local->stub = stub;
+
+                GET_CONTRI_KEY (contri_key, oplocal->loc.parent->gfid, ret);
+                if (ret < 0) {
+                        local->err = ENOMEM;
+                        goto quota_err;
+                }
+
+                /* Removexattr requires uid and gid to be 0,
+                 * reset them in the callback.
+                 */
+                MARKER_SET_UID_GID (frame, local, frame->root);
+
+                newloc.inode = inode_ref (oplocal->loc.inode);
+                newloc.path = gf_strdup (local->loc.path);
+                newloc.name = strrchr (newloc.path, '/');
+                if (newloc.name)
+                        newloc.name++;
+                newloc.parent = inode_ref (local->loc.parent);
+                uuid_copy (newloc.gfid, oplocal->loc.inode->gfid);
+
+                STACK_WIND_COOKIE (frame, marker_rename_release_oldp_lock,
+                                   frame->cookie, FIRST_CHILD(this),
+                                   FIRST_CHILD(this)->fops->removexattr,
+                                   &newloc, contri_key, NULL);
+
+                loc_wipe (&newloc);
+        } else {
+                frame->local = NULL;
+
+                STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
+                                     preoldparent, postoldparent, prenewparent,
+                                     postnewparent, xdata);
+
+                if ((op_ret < 0) || (local == NULL)) {
+                        goto out;
+                }
+
+                if (priv->feature_enabled & GF_XTIME) {
+                        //update marks on oldpath
+                        uuid_copy (local->loc.gfid, oplocal->loc.inode->gfid);
+                        marker_xtime_update_marks (this, oplocal);
+                        marker_xtime_update_marks (this, local);
+                }
+        }
+
+out:
+        if (!(priv->feature_enabled & GF_QUOTA)) {
+                marker_local_unref (local);
+                marker_local_unref (oplocal);
+        }
+
+        return 0;
+
+quota_err:
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_do_rename (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        marker_local_t       *local           = NULL, *oplocal = NULL;
+        char                  contri_key[512] = {0, };
+        int32_t               ret             = 0;
+        int64_t              *contribution    = 0;
+
+        local = frame->local;
+        oplocal = local->oplocal;
+
+        //Reset frame uid and gid if set.
+        if (cookie == (void *) _GF_UID_GID_CHANGED)
+                MARKER_RESET_UID_GID (frame, frame->root, local);
+
+        if ((op_ret < 0) && (op_errno != ENOATTR)) {
+                local->err = op_errno;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "fetching contribution values from %s (gfid:%s) "
+                        "failed (%s)", local->loc.path,
+                        uuid_utoa (local->loc.inode->gfid),
+                        strerror (op_errno));
+                goto err;
+        }
+
+        if (local->loc.inode != NULL) {
+                GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
+                if (ret < 0) {
+                        local->err = errno;
+                        goto err;
+                }
+
+                if (dict_get_bin (dict, contri_key,
+                                  (void **) &contribution) == 0) {
+                        local->contribution = ntoh64 (*contribution);
+                }
+        }
+
+        STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rename, &oplocal->loc,
+                    &local->loc, NULL);
+
+        return 0;
+
+err:
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_get_newpath_contribution (call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        marker_local_t *local           = NULL, *oplocal = NULL;
+        char            contri_key[512] = {0, };
+        int32_t         ret             = 0;
+        int64_t        *contribution    = 0;
+
+        local = frame->local;
+        oplocal = local->oplocal;
+
+        //Reset frame uid and gid if set.
+        if (cookie == (void *) _GF_UID_GID_CHANGED)
+                MARKER_RESET_UID_GID (frame, frame->root, local);
+
+        if ((op_ret < 0) && (op_errno != ENOATTR)) {
+                local->err = op_errno;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "fetching contribution values from %s (gfid:%s) "
+                        "failed (%s)", oplocal->loc.path,
+                        uuid_utoa (oplocal->loc.inode->gfid),
+                        strerror (op_errno));
+                goto err;
+        }
+
+        GET_CONTRI_KEY (contri_key, oplocal->loc.parent->gfid, ret);
+        if (ret < 0) {
+                local->err = errno;
+                goto err;
+        }
+
+        if (dict_get_bin (dict, contri_key, (void **) &contribution) == 0)
+                oplocal->contribution = ntoh64 (*contribution);
+
+        if (local->loc.inode != NULL) {
+                GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
+                if (ret < 0) {
+                        local->err = errno;
+                        goto err;
+                }
+
+                /* getxattr requires uid and gid to be 0,
+                 * reset them in the callback.
+                 */
+                MARKER_SET_UID_GID (frame, local, frame->root);
+                if (uuid_is_null (local->loc.gfid))
+                        uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+                GF_UUID_ASSERT (local->loc.gfid);
+
+                STACK_WIND_COOKIE (frame, marker_do_rename,
+                                   frame->cookie, FIRST_CHILD(this),
+                                   FIRST_CHILD(this)->fops->getxattr,
+                                   &local->loc, contri_key, NULL);
+        } else {
+                marker_do_rename (frame, NULL, this, 0, 0, NULL, NULL);
+        }
+
+        return 0;
+err:
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_get_oldpath_contribution (call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *xdata)
+{
+        marker_local_t *local           = NULL, *oplocal = NULL;
+        char            contri_key[512] = {0, };
+        int32_t         ret             = 0;
+
+        local = frame->local;
+        oplocal = local->oplocal;
+
+        if (op_ret < 0) {
+                local->err = op_errno;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "cannot hold inodelk on %s (gfid:%s) (%s)",
+                        local->next_lock_on->path,
+                        uuid_utoa (local->next_lock_on->inode->gfid),
+                        strerror (op_errno));
+                goto lock_err;
+        }
+
+        GET_CONTRI_KEY (contri_key, oplocal->loc.parent->gfid, ret);
+        if (ret < 0) {
+                local->err = errno;
+                goto quota_err;
+        }
+
+        /* getxattr requires uid and gid to be 0,
+         * reset them in the callback.
+         */
+        MARKER_SET_UID_GID (frame, local, frame->root);
+
+        if (uuid_is_null (oplocal->loc.gfid))
+                        uuid_copy (oplocal->loc.gfid,
+                                   oplocal->loc.inode->gfid);
+
+        GF_UUID_ASSERT (oplocal->loc.gfid);
+
+        STACK_WIND_COOKIE (frame, marker_get_newpath_contribution,
+                           frame->cookie, FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->getxattr,
+                           &oplocal->loc, contri_key, NULL);
+        return 0;
+
+quota_err:
+        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
+        return 0;
+
+lock_err:
+        if ((local->next_lock_on == NULL)
+            || (local->next_lock_on == &local->parent_loc)) {
+                local->next_lock_on = NULL;
+                marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
+        } else {
+                marker_rename_release_newp_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        return 0;
+}
+
+
+int32_t
+marker_rename_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        marker_local_t  *local = NULL, *oplocal = NULL;
+        loc_t           *loc   = NULL;
+        struct gf_flock  lock  = {0, };
+
+        local = frame->local;
+        oplocal = local->oplocal;
+
+        if (op_ret < 0) {
+                if (local->next_lock_on != &oplocal->parent_loc) {
+                        loc = &oplocal->parent_loc;
+                } else {
+                        loc = &local->parent_loc;
+                }
+
+                local->err = op_errno;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "cannot hold inodelk on %s (gfid:%s) (%s)",
+                        loc->path, uuid_utoa (loc->inode->gfid),
+                        strerror (op_errno));
+                goto err;
+        }
+
+        if (local->next_lock_on != NULL) {
+                lock.l_len    = 0;
+                lock.l_start  = 0;
+                lock.l_type   = F_WRLCK;
+                lock.l_whence = SEEK_SET;
+
+                STACK_WIND (frame,
+                            marker_get_oldpath_contribution,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->inodelk,
+                            this->name, local->next_lock_on,
+                            F_SETLKW, &lock, NULL);
+        } else {
+                marker_get_oldpath_contribution (frame, 0, this, 0, 0, NULL);
+        }
+
+        return 0;
+
+err:
+        marker_rename_done (frame, NULL, this, 0, 0, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+        int32_t         ret              = 0;
+        marker_local_t *local            = NULL;
+        marker_local_t *oplocal          = NULL;
+        marker_conf_t  *priv             = NULL;
+        struct gf_flock lock             = {0, };
+        loc_t          *lock_on          = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto rename_wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        oplocal = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, oplocal);
+
+        frame->local = local;
+
+        local->oplocal = marker_local_ref (oplocal);
+
+        ret = loc_copy (&local->loc, newloc);
+        if (ret < 0)
+                goto err;
+
+        ret = loc_copy (&oplocal->loc, oldloc);
+        if (ret < 0)
+                goto err;
+
+        if (!(priv->feature_enabled & GF_QUOTA)) {
+                goto rename_wind;
+        }
+
+        ret = mq_inode_loc_fill (NULL, newloc->parent, &local->parent_loc);
+        if (ret < 0)
+                goto err;
+
+        ret = mq_inode_loc_fill (NULL, oldloc->parent, &oplocal->parent_loc);
+        if (ret < 0)
+                goto err;
+
+        if ((newloc->inode != NULL) && (newloc->parent != oldloc->parent)
+            && (uuid_compare (newloc->parent->gfid,
+                              oldloc->parent->gfid) < 0)) {
+                lock_on = &local->parent_loc;
+                local->next_lock_on = &oplocal->parent_loc;
+        } else {
+                lock_on = &oplocal->parent_loc;
+                if ((newloc->inode != NULL) && (newloc->parent
+                                                != oldloc->parent)) {
+                        local->next_lock_on = &local->parent_loc;
+                }
+        }
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND (frame,
+                    marker_rename_inodelk_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, lock_on,
+                    F_SETLKW, &lock, NULL);
+
+        return 0;
+
+rename_wind:
+        STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM, NULL,
+                             NULL, NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "truncating a file ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_truncate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "truncating a file ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_ftruncate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+        marker_conf_t      *priv    = NULL;
+        marker_local_t     *local   = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "creating symlinks ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_set_inode_xattr (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int
+marker_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+                loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_symlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                    xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (symlink, frame, -1, ENOMEM, NULL,
+                             NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "creating symlinks ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode,
+                             buf, preparent, postparent, xdata);
+
+        if (op_ret == -1 ||  local == NULL)
+                goto out;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+        priv = this->private;
+
+        if ((priv->feature_enabled & GF_QUOTA) && (S_ISREG (local->mode))) {
+                mq_set_inode_xattr (this, &local->loc);
+        }
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int
+marker_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        local->mode = mode;
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_mknod_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                    xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (mknod, frame, -1, ENOMEM, NULL,
+                             NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+marker_fallocate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "fallocating a file ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (fallocate, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_fallocate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		    xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred during discard",
+                        strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_discard_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (discard, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int32_t
+marker_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred during zerofill",
+                        strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA)
+                mq_initiate_quota_txn (this, &local->loc);
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               size_t len, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_zerofill_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+/* when a call from the special client is received on
+ * key trusted.glusterfs.volume-mark with value "RESET"
+ * or if the value is 0length, update the change the
+ * access time and modification time via touching the
+ * timestamp file.
+ */
+int32_t
+call_from_sp_client_to_reset_tmfile (call_frame_t *frame,
+                                     xlator_t     *this,
+                                     dict_t       *dict)
+{
+        int32_t          fd       = 0;
+        int32_t          op_ret   = 0;
+        int32_t          op_errno = 0;
+        data_t          *data     = NULL;
+        marker_conf_t   *priv     = NULL;
+
+        if (frame == NULL || this == NULL || dict == NULL)
+                return -1;
+
+        priv = this->private;
+
+        data = dict_get (dict, "trusted.glusterfs.volume-mark");
+        if (data == NULL)
+                return -1;
+
+        if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+                op_ret = -1;
+                op_errno = EPERM;
+
+                goto out;
+        }
+
+        if (data->len == 0 || (data->len == 5 &&
+                               memcmp (data->data, "RESET", 5) == 0)) {
+                fd = open (priv->timestamp_file, O_WRONLY|O_TRUNC);
+                if (fd != -1) {
+                        /* TODO check  whether the O_TRUNC would update the
+                         * timestamps on a zero length file on all machies.
+                         */
+                        close (fd);
+                }
+
+                if (fd != -1 || errno == ENOENT) {
+                        op_ret = 0;
+                        op_errno = 0;
+                } else {
+                        op_ret = -1;
+                        op_errno = errno;
+                }
+        } else {
+                op_ret = -1;
+                op_errno = EINVAL;
+        }
+out:
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred in "
+                        "setxattr ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        ret = call_from_sp_client_to_reset_tmfile (frame, this, dict);
+        if (ret == 0)
+                return 0;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
+                        "creating symlinks ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int32_t flags, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        ret = call_from_sp_client_to_reset_tmfile (frame, this, dict);
+        if (ret == 0)
+                return 0;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_fsetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "%s occurred while "
+                        "creating symlinks ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
+                             statpost, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+
+int32_t
+marker_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_fsetattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, ((op_errno == ENOENT) ? GF_LOG_DEBUG :
+                                     GF_LOG_ERROR),
+                        "%s occurred during setattr of %s",
+                        strerror (op_errno),
+                        (local ? local->loc.path : "<nul>"));
+        }
+
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
+                             statpost, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_setattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        marker_local_t     *local   = NULL;
+        marker_conf_t      *priv    = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "%s occurred while "
+                        "creating symlinks ", strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_XTIME)
+                marker_xtime_update_marks (this, local);
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
+{
+        int32_t          ret   = 0;
+        marker_local_t  *local = NULL;
+        marker_conf_t   *priv  = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+
+        if (ret == -1)
+                goto err;
+wind:
+        STACK_WIND (frame, marker_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM, NULL);
+
+        return 0;
+}
+
+
+int32_t
+marker_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
+{
+        marker_conf_t  *priv    = NULL;
+        marker_local_t *local   = NULL;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_TRACE, "lookup failed with %s",
+                        strerror (op_errno));
+        }
+
+        local = (marker_local_t *) frame->local;
+
+        frame->local = NULL;
+
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             dict, postparent);
+
+        if (op_ret == -1 || local == NULL)
+                goto out;
+
+        /* copy the gfid from the stat structure instead of inode,
+         * since if the lookup is fresh lookup, then the inode
+         * would have not yet linked to the inode table which happens
+         * in protocol/server.
+         */
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+
+        priv = this->private;
+
+        if (priv->feature_enabled & GF_QUOTA) {
+                mq_xattr_state (this, &local->loc, dict, *buf);
+        }
+
+out:
+        marker_local_unref (local);
+
+        return 0;
+}
+
+int32_t
+marker_lookup (call_frame_t *frame, xlator_t *this,
+               loc_t *loc, dict_t *xattr_req)
+{
+        int32_t         ret     = 0;
+        marker_local_t *local   = NULL;
+        marker_conf_t  *priv    = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        local = mem_get0 (this->local_pool);
+
+        MARKER_INIT_LOCAL (frame, local);
+
+        ret = loc_copy (&local->loc, loc);
+        if (ret == -1)
+                goto err;
+
+        if ((priv->feature_enabled & GF_QUOTA) && xattr_req)
+                mq_req_xattr (this, loc, xattr_req);
+wind:
+        STACK_WIND (frame, marker_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (lookup, frame, -1, 0, NULL, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+marker_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
+{
+        gf_dirent_t *entry = NULL;
+
+        if (op_ret <= 0)
+                goto unwind;
+
+        list_for_each_entry (entry, &entries->list, list) {
+                /* TODO: fill things */
+        }
+
+unwind:
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+
+        return 0;
+}
+
+int
+marker_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, dict_t *dict)
+{
+        marker_conf_t  *priv    = NULL;
+
+        priv = this->private;
+
+        if (priv->feature_enabled == 0)
+                goto wind;
+
+        if ((priv->feature_enabled & GF_QUOTA) && dict)
+                mq_req_xattr (this, NULL, dict);
+
+wind:
+        STACK_WIND (frame, marker_readdirp_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
+                    fd, size, offset, dict);
+
+        return 0;
+}
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_marker_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+                       "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+
+int32_t
+init_xtime_priv (xlator_t *this, dict_t *options)
+{
+        data_t          *data    = NULL;
+        int32_t          ret     = -1;
+        marker_conf_t   *priv    = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, options, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+
+        priv = this->private;
+
+        if((data = dict_get (options, VOLUME_UUID)) != NULL) {
+                priv->volume_uuid = data->data;
+
+                ret = uuid_parse (priv->volume_uuid, priv->volume_uuid_bin);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid volume uuid %s", priv->volume_uuid);
+                        goto out;
+                }
+
+                ret = gf_asprintf (& (priv->marker_xattr), "%s.%s.%s",
+                                   MARKER_XATTR_PREFIX, priv->volume_uuid,
+                                   XTIME);
+
+                if (ret == -1){
+                        priv->marker_xattr = NULL;
+
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to allocate memory");
+                        goto out;
+                }
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "the volume-uuid = %s", priv->volume_uuid);
+        } else {
+                priv->volume_uuid = NULL;
+
+                gf_log (this->name, GF_LOG_ERROR,
+                        "please specify the volume-uuid"
+                        "in the translator options");
+
+                return -1;
+        }
+
+        if ((data = dict_get (options, TIMESTAMP_FILE)) != NULL) {
+                priv->timestamp_file = data->data;
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "the timestamp-file is = %s",
+                        priv->timestamp_file);
+
+        } else {
+                priv->timestamp_file = NULL;
+
+                gf_log (this->name, GF_LOG_ERROR,
+                        "please specify the timestamp-file"
+                        "in the translator options");
+
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+marker_xtime_priv_cleanup (xlator_t *this)
+{
+        marker_conf_t *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+
+        priv = (marker_conf_t *) this->private;
+
+        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+
+        GF_FREE (priv->volume_uuid);
+
+        GF_FREE (priv->timestamp_file);
+
+        GF_FREE (priv->marker_xattr);
+out:
+        return;
+}
+
+void
+marker_priv_cleanup (xlator_t *this)
+{
+        marker_conf_t *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+
+        priv = (marker_conf_t *) this->private;
+
+        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+
+        marker_xtime_priv_cleanup (this);
+
+        LOCK_DESTROY (&priv->lock);
+
+        GF_FREE (priv);
+out:
+        return;
+}
+
+int32_t
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int32_t         ret     = 0;
+        data_t         *data    = NULL;
+        gf_boolean_t    flag    = _gf_false;
+        marker_conf_t  *priv    = NULL;
+
+        GF_ASSERT (this);
+        GF_ASSERT (this->private);
+
+        priv = this->private;
+
+        priv->feature_enabled = 0;
+
+        GF_VALIDATE_OR_GOTO (this->name, options, out);
+
+        data = dict_get (options, "quota");
+        if (data) {
+                ret = gf_string2boolean (data->data, &flag);
+                if (ret == 0 && flag == _gf_true) {
+                        ret = init_quota_priv (this);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to initialize quota private");
+                        } else {
+                                priv->feature_enabled |= GF_QUOTA;
+                        }
+                }
+        }
+
+        data = dict_get (options, "xtime");
+        if (data) {
+                ret = gf_string2boolean (data->data, &flag);
+                if (ret == 0 && flag == _gf_true) {
+                        marker_xtime_priv_cleanup (this);
+
+                        ret = init_xtime_priv (this, options);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to initialize xtime private, "
+                                        "xtime updation will fail");
+                        } else {
+                                priv->feature_enabled |= GF_XTIME;
+                                data = dict_get (options, "gsync-force-xtime");
+                                if (!data)
+                                        goto out;
+                                ret = gf_string2boolean (data->data, &flag);
+                                if (ret == 0 && flag)
+                                        priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
+                        }
+                }
+        }
+out:
+        return ret;
+}
+
+
+int32_t
+init (xlator_t *this)
+{
+        dict_t        *options = NULL;
+        data_t        *data    = NULL;
+        int32_t        ret     = 0;
+        gf_boolean_t   flag    = _gf_false;
+        marker_conf_t *priv    = NULL;
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "marker translator needs subvolume defined.");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Volume is dangling.");
+                return -1;
+        }
+
+        options = this->options;
+
+        ALLOCATE_OR_GOTO (this->private, marker_conf_t, err);
+
+        priv = this->private;
+
+        priv->feature_enabled = 0;
+
+        LOCK_INIT (&priv->lock);
+
+        data = dict_get (options, "quota");
+        if (data) {
+                ret = gf_string2boolean (data->data, &flag);
+                if (ret == 0 && flag == _gf_true) {
+                        ret = init_quota_priv (this);
+                        if (ret < 0)
+                                goto err;
+
+                        priv->feature_enabled |= GF_QUOTA;
+                }
+        }
+
+        data = dict_get (options, "xtime");
+        if (data) {
+                ret = gf_string2boolean (data->data, &flag);
+                if (ret == 0 && flag == _gf_true) {
+                        ret = init_xtime_priv (this, options);
+                        if (ret < 0)
+                                goto err;
+
+                        priv->feature_enabled |= GF_XTIME;
+                        data = dict_get (options, "gsync-force-xtime");
+                        if (!data)
+                                goto cont;
+                        ret = gf_string2boolean (data->data, &flag);
+                        if (ret == 0 && flag)
+                                priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
+                }
+        }
+
+ cont:
+        this->local_pool = mem_pool_new (marker_local_t, 128);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto err;
+        }
+
+        return 0;
+err:
+        marker_priv_cleanup (this);
+
+        return -1;
+}
+
+int32_t
+marker_forget (xlator_t *this, inode_t *inode)
+{
+        marker_inode_ctx_t *ctx   = NULL;
+        uint64_t            value = 0;
+
+        if (inode_ctx_del (inode, this, &value) != 0)
+                goto out;
+
+        ctx = (marker_inode_ctx_t *)(unsigned long)value;
+        if (ctx == NULL) {
+                goto out;
+        }
+
+        mq_forget (this, ctx->quota_ctx);
+
+        GF_FREE (ctx);
+out:
+        return 0;
+}
+
+void
+fini (xlator_t *this)
+{
+        marker_priv_cleanup (this);
+}
+
+struct xlator_fops fops = {
+        .lookup      = marker_lookup,
+        .create      = marker_create,
+        .mkdir       = marker_mkdir,
+        .writev      = marker_writev,
+        .truncate    = marker_truncate,
+        .ftruncate   = marker_ftruncate,
+        .symlink     = marker_symlink,
+        .link        = marker_link,
+        .unlink      = marker_unlink,
+        .rmdir       = marker_rmdir,
+        .rename      = marker_rename,
+        .mknod       = marker_mknod,
+        .setxattr    = marker_setxattr,
+        .fsetxattr   = marker_fsetxattr,
+        .setattr     = marker_setattr,
+        .fsetattr    = marker_fsetattr,
+        .removexattr = marker_removexattr,
+        .getxattr    = marker_getxattr,
+        .readdirp    = marker_readdirp,
+	.fallocate   = marker_fallocate,
+	.discard     = marker_discard,
+        .zerofill    = marker_zerofill,
+};
+
+struct xlator_cbks cbks = {
+        .forget = marker_forget
+};
+
+struct volume_options options[] = {
+        {.key = {"volume-uuid"}},
+        {.key = {"timestamp-file"}},
+        {.key = {"quota"}},
+        {.key = {"xtime"}},
+        {.key = {"gsync-force-xtime"}},
+        {.key = {NULL}}
+};
diff --git a/xlators/features/marker/src/marker.h b/xlators/features/marker/src/marker.h
new file mode 100644
index 000000000..1a58f8cfc
--- /dev/null
+++ b/xlators/features/marker/src/marker.h
@@ -0,0 +1,138 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _MARKER_H
+#define _MARKER_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "marker-quota.h"
+#include "xlator.h"
+#include "defaults.h"
+#include "uuid.h"
+#include "call-stub.h"
+
+#define MARKER_XATTR_PREFIX "trusted.glusterfs"
+#define XTIME               "xtime"
+#define VOLUME_MARK         "volume-mark"
+#define VOLUME_UUID         "volume-uuid"
+#define TIMESTAMP_FILE      "timestamp-file"
+
+enum {
+        GF_QUOTA             = 1,
+        GF_XTIME             = 2,
+        GF_XTIME_GSYNC_FORCE = 4,
+};
+
+/*initialize the local variable*/
+#define MARKER_INIT_LOCAL(_frame,_local) do {                   \
+                _frame->local = _local;                         \
+                _local->pid = _frame->root->pid;                \
+                memset (&_local->loc, 0, sizeof (loc_t));       \
+                _local->ref = 1;                                \
+                _local->uid = -1;                               \
+                _local->gid = -1;                               \
+                LOCK_INIT (&_local->lock);                      \
+                _local->oplocal = NULL;                         \
+        } while (0)
+
+/* try alloc and if it fails, goto label */
+#define ALLOCATE_OR_GOTO(var, type, label) do {                  \
+                var = GF_CALLOC (sizeof (type), 1,               \
+                                 gf_marker_mt_##type);           \
+                if (!var) {                                      \
+                        gf_log (this->name, GF_LOG_ERROR,        \
+                                "out of memory :(");             \
+                        goto label;                              \
+                }                                                \
+        } while (0)
+
+#define _MARKER_SET_UID_GID(dest, src)          \
+        do {                                    \
+                if (src->uid != -1 &&           \
+                    src->gid != -1) {           \
+                        dest->uid = src->uid;   \
+                        dest->gid = src->gid;   \
+                }                               \
+        } while (0)
+
+#define MARKER_SET_UID_GID(frame, dest, src)                    \
+        do {                                                    \
+                _MARKER_SET_UID_GID (dest, src);                \
+                frame->root->uid = 0;                           \
+                frame->root->gid = 0;                           \
+                frame->cookie = (void *) _GF_UID_GID_CHANGED;   \
+        } while (0)
+
+#define MARKER_RESET_UID_GID(frame, dest, src)                  \
+        do {                                                    \
+                _MARKER_SET_UID_GID (dest, src);                \
+                frame->cookie = NULL;                           \
+        } while (0)
+
+struct marker_local{
+        uint32_t        timebuf[2];
+        pid_t           pid;
+        loc_t           loc;
+        loc_t           parent_loc;
+        loc_t          *next_lock_on;
+        uid_t           uid;
+        gid_t           gid;
+        int32_t         ref;
+        int32_t         ia_nlink;
+        gf_lock_t       lock;
+        mode_t          mode;
+        int32_t         err;
+        call_stub_t    *stub;
+        int64_t         contribution;
+        struct marker_local *oplocal;
+
+        /* marker quota specific */
+        int64_t delta;
+        int64_t d_off;
+        int64_t sum;
+        int64_t size;
+        int32_t hl_count;
+        int32_t dentry_child_count;
+
+        fd_t         *fd;
+        call_frame_t *frame;
+
+        quota_inode_ctx_t    *ctx;
+        inode_contribution_t *contri;
+
+        int xflag;
+        dict_t *xdata;
+};
+typedef struct marker_local marker_local_t;
+
+#define quota_local_t marker_local_t
+
+struct marker_inode_ctx {
+        struct quota_inode_ctx *quota_ctx;
+};
+typedef struct marker_inode_ctx marker_inode_ctx_t;
+
+struct marker_conf{
+        char         feature_enabled;
+        char        *size_key;
+        char        *dirty_key;
+        char        *volume_uuid;
+        uuid_t      volume_uuid_bin;
+        char        *timestamp_file;
+        char        *marker_xattr;
+        uint64_t     quota_lk_owner;
+        gf_lock_t    lock;
+};
+typedef struct marker_conf marker_conf_t;
+
+#endif
diff --git a/xlators/features/path-convertor/src/Makefile.am b/xlators/features/path-convertor/src/Makefile.am
index 1fde19352..393a7bd08 100644
--- a/xlators/features/path-convertor/src/Makefile.am
+++ b/xlators/features/path-convertor/src/Makefile.am
@@ -1,14 +1,15 @@
 
 xlator_LTLIBRARIES = path-converter.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-path_converter_la_LDFLAGS = -module -avoidversion 
+path_converter_la_LDFLAGS = -module -avoid-version 
 
 path_converter_la_SOURCES = path.c
 path_converter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/path-convertor/src/path-mem-types.h b/xlators/features/path-convertor/src/path-mem-types.h
new file mode 100644
index 000000000..77ada8d53
--- /dev/null
+++ b/xlators/features/path-convertor/src/path-mem-types.h
@@ -0,0 +1,22 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __PATH_MEM_TYPES_H__
+#define __PATH_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_path_mem_types_ {
+        gf_path_mt_path_private_t = gf_common_mt_end + 1,
+        gf_path_mt_char,
+        gf_path_mt_regex_t,
+        gf_path_mt_end
+};
+#endif
+
diff --git a/xlators/features/path-convertor/src/path.c b/xlators/features/path-convertor/src/path.c
index 41ef1d8a8..5c52e0a8d 100644
--- a/xlators/features/path-convertor/src/path.c
+++ b/xlators/features/path-convertor/src/path.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 /* TODO: add gf_log to all the cases returning errors */
 
 #ifndef _CONFIG_H
@@ -35,6 +25,7 @@
 #include <errno.h>
 #include "glusterfs.h"
 #include "xlator.h"
+#include "path-mem-types.h"
 
 typedef struct path_private
 {
@@ -51,7 +42,7 @@ static char *
 name_this_to_that (xlator_t *xl, const char *path, const char *name)
 {
 	path_private_t *priv = xl->private;
-	char priv_path[ZR_PATH_MAX] = {0,};
+	char priv_path[PATH_MAX] = {0,};
 	char *tmp_name = NULL;
 	int32_t path_len = strlen (path);
 	int32_t name_len = strlen (name) - ZR_FILE_CONTENT_STRLEN;
@@ -63,7 +54,9 @@ name_this_to_that (xlator_t *xl, const char *path, const char *name)
 
 	if (priv->end_off && (total_len > priv->end_off)) {
 		j = priv->start_off;
-		tmp_name = CALLOC (1, (total_len + ZR_FILE_CONTENT_STRLEN));
+		tmp_name = GF_CALLOC (1, (total_len +
+                                          ZR_FILE_CONTENT_STRLEN),
+                                      gf_path_mt_char);
 		ERR_ABORT (tmp_name);
 
 		/* Get the complete path for the file first */
@@ -104,7 +97,7 @@ path_this_to_that (xlator_t *xl, const char *path)
 	int32_t i = 0, j = 0;
 
 	if (priv->end_off && (path_len > priv->start_off)) {
-		priv_path = CALLOC (1, path_len);
+		priv_path = GF_CALLOC (1, path_len, gf_path_mt_char);
 		ERR_ABORT (priv_path);
 
 		if (priv->start_off && (path_len > priv->start_off))
@@ -134,7 +127,9 @@ path_create_cbk (call_frame_t *frame,
 		 int32_t op_errno,
 		 fd_t *fd,
 		 inode_t *inode,
-		 struct stat *buf)
+		 struct iatt *buf,
+                 struct iatt *preparent,
+                 struct iatt *postparent)
 {
 	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
 	return 0;
@@ -184,9 +179,10 @@ path_readlink_cbk (call_frame_t *frame,
 		   xlator_t *this,
 		   int32_t op_ret,
 		   int32_t op_errno,
-		   const char *buf)
+		   const char *buf,
+                   struct iatt *sbuf)
 {
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
+	STACK_UNWIND (frame, op_ret, op_errno, buf, sbuf);
 	return 0;
 }
 
@@ -197,8 +193,9 @@ path_lookup_cbk (call_frame_t *frame,
 		 int32_t op_ret,
 		 int32_t op_errno,
 		 inode_t *inode,
-		 struct stat *buf,
-		 dict_t *xattr)
+		 struct iatt *buf,
+		 dict_t *xattr,
+                 struct iatt *postparent)
 {
 	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, xattr);
 	return 0;
@@ -212,7 +209,9 @@ path_symlink_cbk (call_frame_t *frame,
 		  int32_t op_ret,
 		  int32_t op_errno,
 		  inode_t *inode,
-		  struct stat *buf)
+                  struct iatt *buf,
+                  struct iatt *preparent,
+                  struct iatt *postparent)
 {
 	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
 	return 0;
@@ -225,7 +224,9 @@ path_mknod_cbk (call_frame_t *frame,
 		int32_t op_ret,
 		int32_t op_errno,
 		inode_t *inode,
-		struct stat *buf)
+                struct iatt *buf,
+                struct iatt *preparent,
+                struct iatt *postparent)
 {
 	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
 	return 0;
@@ -239,7 +240,9 @@ path_mkdir_cbk (call_frame_t *frame,
 		int32_t op_ret,
 		int32_t op_errno,
 		inode_t *inode,
-		struct stat *buf)
+                struct iatt *buf,
+                struct iatt *preparent,
+                struct iatt *postparent)
 {
 	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
 	return 0;
@@ -252,7 +255,9 @@ path_link_cbk (call_frame_t *frame,
 	       int32_t op_ret,
 	       int32_t op_errno,
 	       inode_t *inode,
-	       struct stat *buf)
+               struct iatt *buf,
+               struct iatt *preparent,
+               struct iatt *postparent)
 {
 	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
 	return 0;
@@ -271,13 +276,31 @@ path_opendir_cbk (call_frame_t *frame,
 }
 
 
-int32_t 
+int32_t
+path_rename_buf_cbk (call_frame_t *frame,
+		     void *cookie,
+		     xlator_t *this,
+		     int32_t op_ret,
+		     int32_t op_errno,
+		     struct iatt *buf,
+                     struct iatt *preoldparent,
+                     struct iatt *postoldparent,
+                     struct iatt *prenewparent,
+                     struct iatt *postnewparent)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, buf);
+	return 0;
+}
+
+
+
+int32_t
 path_common_buf_cbk (call_frame_t *frame,
 		     void *cookie,
 		     xlator_t *this,
 		     int32_t op_ret,
 		     int32_t op_errno,
-		     struct stat *buf)
+		     struct iatt *buf)
 {
 	STACK_UNWIND (frame, op_ret, op_errno, buf);
 	return 0;
@@ -296,6 +319,25 @@ path_common_dict_cbk (call_frame_t *frame,
 }
 
 int32_t 
+path_common_remove_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,struct iatt *preparent,
+                        struct iatt *postparent)
+{
+	STACK_UNWIND (frame, op_ret, op_errno);
+	return 0;
+}
+
+int32_t
+path_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,struct iatt *prebuf,
+                   struct iatt *postbuf)
+{
+	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
+	return 0;
+}
+
+
+int32_t
 path_common_cbk (call_frame_t *frame,
 		 void *cookie,
 		 xlator_t *this,
@@ -329,7 +371,7 @@ path_lookup (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -356,7 +398,7 @@ path_stat (call_frame_t *frame,
   
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -385,7 +427,7 @@ path_readlink (call_frame_t *frame,
   
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -416,7 +458,7 @@ path_mknod (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -445,7 +487,7 @@ path_mkdir (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -465,14 +507,14 @@ path_unlink (call_frame_t *frame,
 	loc->path = tmp_path;
 
 	STACK_WIND (frame, 
-		    path_common_cbk, 
+		    path_common_remove_cbk,
 		    FIRST_CHILD(this), 
 		    FIRST_CHILD(this)->fops->unlink, 
 		    loc);
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -492,14 +534,14 @@ path_rmdir (call_frame_t *frame,
 	loc->path = tmp_path;
 
 	STACK_WIND (frame, 
-		    path_common_cbk,
+		    path_common_remove_cbk,
 		    FIRST_CHILD(this), 
 		    FIRST_CHILD(this)->fops->rmdir, 
 		    loc);
   
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -528,7 +570,7 @@ path_symlink (call_frame_t *frame,
   
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -558,7 +600,7 @@ path_rename (call_frame_t *frame,
 	newloc->path = tmp_newloc_path;
 
 	STACK_WIND (frame, 
-		    path_common_buf_cbk,
+		    path_rename_buf_cbk,
 		    FIRST_CHILD(this), 
 		    FIRST_CHILD(this)->fops->rename, 
 		    oldloc,
@@ -566,11 +608,11 @@ path_rename (call_frame_t *frame,
   
 	oldloc->path = oldloc_path;	
 	if (tmp_oldloc_path != oldloc_path)
-		FREE (tmp_oldloc_path);
+		GF_FREE (tmp_oldloc_path);
 
 	newloc->path = newloc_path;	
 	if (tmp_newloc_path != newloc_path)
-		FREE (tmp_newloc_path);
+		GF_FREE (tmp_newloc_path);
 
 	return 0;
 }
@@ -608,75 +650,59 @@ path_link (call_frame_t *frame,
 
 	oldloc->path = oldloc_path;	
 	if (tmp_oldloc_path != oldloc_path)
-		FREE (tmp_oldloc_path);
+		GF_FREE (tmp_oldloc_path);
 
 	newloc->path = newloc_path;	
 	if (tmp_newloc_path != newloc_path)
-		FREE (tmp_newloc_path);
+		GF_FREE (tmp_newloc_path);
 
 	return 0;
 }
 
 int32_t 
-path_chmod (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    mode_t mode)
+path_setattr_cbk (call_frame_t *frame,
+                  void *cookie,
+                  xlator_t *this,
+                  int32_t op_ret,
+                  int32_t op_errno,
+                  struct iatt *preop,
+                  struct iatt *postop)
 {
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_buf_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->chmod, 
-		    loc, 
-		    mode);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		FREE (tmp_path);
-
+	STACK_UNWIND (frame, op_ret, op_errno, preop, postop);
 	return 0;
 }
 
-int32_t 
-path_chown (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    uid_t uid,
-	    gid_t gid)
+int32_t
+path_setattr (call_frame_t *frame,
+              xlator_t *this,
+              loc_t *loc,
+              struct iatt *stbuf,
+              int32_t valid)
 {
 	char *loc_path = (char *)loc->path;
 	char *tmp_path = NULL;
-	
+
 	if (!(tmp_path = path_this_to_that (this, loc->path))) {
 		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
 		return 0;
 	}
 	loc->path = tmp_path;
 
-	STACK_WIND (frame, 
-		    path_common_buf_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->chown, 
-		    loc, 
-		    uid,
-		    gid);
+	STACK_WIND (frame,
+		    path_setattr_cbk,
+		    FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->setattr,
+		    loc,
+		    stbuf, valid);
 
-	loc->path = loc_path;	
+	loc->path = loc_path;
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
 
+
 int32_t 
 path_truncate (call_frame_t *frame,
 	       xlator_t *this,
@@ -693,7 +719,7 @@ path_truncate (call_frame_t *frame,
 	loc->path = tmp_path;
 
 	STACK_WIND (frame, 
-		    path_common_buf_cbk, 
+		    path_truncate_cbk,
 		    FIRST_CHILD(this), 
 		    FIRST_CHILD(this)->fops->truncate, 
 		    loc, 
@@ -701,46 +727,19 @@ path_truncate (call_frame_t *frame,
   
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
 
-int32_t 
-path_utimens (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      struct timespec tv[2])
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_buf_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->utimens, 
-		    loc, 
-		    tv);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		FREE (tmp_path);
-
-	return 0;
-}
 
 int32_t 
 path_open (call_frame_t *frame,
 	   xlator_t *this,
 	   loc_t *loc,
 	   int32_t flags,
-	   fd_t *fd)
+	   fd_t *fd,
+           int32_t wbflags)
 {
 	char *loc_path = (char *)loc->path;
 	char *tmp_path = NULL;
@@ -757,11 +756,12 @@ path_open (call_frame_t *frame,
 		    FIRST_CHILD(this)->fops->open, 
 		    loc, 
 		    flags,
-		    fd);
+		    fd,
+                    wbflags);
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -794,7 +794,7 @@ path_create (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -836,10 +836,9 @@ path_setxattr (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
-	if (tmp_name)
-		FREE (tmp_name);
+	GF_FREE (tmp_name);
 
 	return 0;
 }
@@ -873,10 +872,10 @@ path_getxattr (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	if (tmp_name != name)
-		FREE (tmp_name);
+		GF_FREE (tmp_name);
 
 	return 0;
 }
@@ -910,10 +909,10 @@ path_removexattr (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	if (tmp_name != name)
-		FREE (tmp_name);
+		GF_FREE (tmp_name);
 
 	return 0;
 }
@@ -942,7 +941,7 @@ path_opendir (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -971,7 +970,7 @@ path_access (call_frame_t *frame,
 
 	loc->path = loc_path;
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -1013,7 +1012,7 @@ path_checksum (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -1021,7 +1020,7 @@ path_checksum (call_frame_t *frame,
 
 int32_t
 path_entrylk (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *basename,
+	      const char *volume, loc_t *loc, const char *basename,
 	      entrylk_cmd cmd, entrylk_type type)
 {
 	char *loc_path = (char *)loc->path;
@@ -1036,18 +1035,18 @@ path_entrylk (call_frame_t *frame, xlator_t *this,
 	STACK_WIND (frame, path_common_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->entrylk,
-		    loc, basename, cmd, type);
+		    volume, loc, basename, cmd, type);
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
 
 int32_t
 path_inodelk (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, int32_t cmd, struct flock *lock)
+              const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
 {
 	char *loc_path = (char *)loc->path;
 	char *tmp_path = NULL;
@@ -1062,11 +1061,11 @@ path_inodelk (call_frame_t *frame, xlator_t *this,
 		    path_common_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->inodelk,
-		    loc, cmd, lock);
+		    volume, loc, cmd, lock);
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
@@ -1098,11 +1097,29 @@ path_xattrop (call_frame_t *frame,
 
 	loc->path = loc_path;	
 	if (tmp_path != loc_path)
-		FREE (tmp_path);
+		GF_FREE (tmp_path);
 
 	return 0;
 }
 
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_path_mt_end + 1);
+        
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        "failed");
+                return ret;
+        }
+
+        return ret;
+}
 
 int32_t 
 init (xlator_t *this)
@@ -1121,7 +1138,7 @@ init (xlator_t *this)
 			"dangling volume. check volfile ");
 	}
   
-	priv = CALLOC (1, sizeof (*priv));
+	priv = GF_CALLOC (1, sizeof (*priv), gf_path_mt_path_private_t);
 	ERR_ABORT (priv);
 	if (dict_get (options, "start-offset")) {
 		priv->start_off = data_to_int32 (dict_get (options, 
@@ -1134,7 +1151,8 @@ init (xlator_t *this)
 
 	if (dict_get (options, "regex")) {
 		int32_t ret = 0;
-		priv->preg = CALLOC (1, sizeof (regex_t));
+		priv->preg = GF_CALLOC (1, sizeof (regex_t),
+                                        gf_path_mt_regex_t);
 		ERR_ABORT (priv->preg);
 		ret = regcomp (priv->preg, 
 			       data_to_str (dict_get (options, "regex")), 
@@ -1142,7 +1160,7 @@ init (xlator_t *this)
 		if (ret) {
 			gf_log (this->name, GF_LOG_ERROR, 
 				"Failed to compile the 'option regex'");
-			FREE (priv);
+			GF_FREE (priv);
 			return -1;
 		}
 		if (dict_get (options, "replace-with")) {
@@ -1173,10 +1191,7 @@ struct xlator_fops fops = {
 	.symlink     = path_symlink,
 	.rename      = path_rename,
 	.link        = path_link,
-	.chmod       = path_chmod,
-	.chown       = path_chown,
 	.truncate    = path_truncate,
-	.utimens     = path_utimens,
 	.open        = path_open,
 	.setxattr    = path_setxattr,
 	.getxattr    = path_getxattr,
@@ -1189,13 +1204,9 @@ struct xlator_fops fops = {
 	.xattrop     = path_xattrop,
 	.entrylk     = path_entrylk,
 	.inodelk     = path_inodelk,
+        .setattr     = path_setattr,
 };
 
-
-struct xlator_mops mops = {
-};
-
-
 struct xlator_cbks cbks = {
 };
 
diff --git a/xlators/features/protect/Makefile.am b/xlators/features/protect/Makefile.am
new file mode 100644
index 000000000..d471a3f92
--- /dev/null
+++ b/xlators/features/protect/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES = 
diff --git a/xlators/features/protect/src/Makefile.am b/xlators/features/protect/src/Makefile.am
new file mode 100644
index 000000000..7eb93f32e
--- /dev/null
+++ b/xlators/features/protect/src/Makefile.am
@@ -0,0 +1,21 @@
+xlator_LTLIBRARIES = prot_dht.la prot_client.la prot_server.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+prot_dht_la_LDFLAGS = -module -avoidversion
+prot_dht_la_SOURCES = prot_dht.c
+prot_dht_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+prot_client_la_LDFLAGS = -module -avoidversion
+prot_client_la_SOURCES = prot_client.c
+prot_client_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+prot_server_la_LDFLAGS = -module -avoidversion
+prot_server_la_SOURCES = prot_server.c
+prot_server_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES = 
+
diff --git a/xlators/features/protect/src/prot_client.c b/xlators/features/protect/src/prot_client.c
new file mode 100644
index 000000000..a27216d0a
--- /dev/null
+++ b/xlators/features/protect/src/prot_client.c
@@ -0,0 +1,215 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+#include <execinfo.h>
+
+#define NUM_FRAMES 20
+
+static char PROTECT_KEY[] = "trusted.glusterfs.protect";
+
+enum {
+        PROT_ACT_NONE = 0,
+        PROT_ACT_LOG,
+        PROT_ACT_REJECT,
+};
+
+void
+pcli_print_trace (char *name, call_frame_t *frame)
+{
+        void    *frames[NUM_FRAMES];
+        char    **symbols;
+        int     size;
+        int     i;
+
+        gf_log (name, GF_LOG_INFO, "Translator stack:");
+        while (frame) {
+                gf_log (name, GF_LOG_INFO, "%s (%s)",
+                        frame->wind_from, frame->this->name);
+                frame = frame->next;
+        }
+
+        size = backtrace(frames,NUM_FRAMES);
+        if (size <= 0) {
+                return;
+        }
+        symbols = backtrace_symbols(frames,size);
+        if (!symbols) {
+                return;
+        }
+
+        gf_log(name, GF_LOG_INFO, "Processor stack:");
+        for (i = 0; i < size; ++i) {
+                gf_log (name, GF_LOG_INFO, "%s", symbols[i]);
+        }
+        free(symbols);
+}
+
+int32_t
+pcli_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+        uint64_t        value;
+
+        if (newloc->parent == oldloc->parent) {
+                gf_log (this->name, GF_LOG_DEBUG, "rename in same directory");
+                goto simple_unwind;
+        }
+        if (!oldloc->parent) {
+                goto simple_unwind;
+        }
+        if (inode_ctx_get(oldloc->parent,this,&value) != 0) {
+                goto simple_unwind;
+        }
+
+        if (value != PROT_ACT_NONE) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "got rename for protected %s", oldloc->path);
+                pcli_print_trace(this->name,frame->next);
+                if (value == PROT_ACT_REJECT) {
+                        STACK_UNWIND_STRICT (rename, frame, -1, EPERM,
+                                             NULL, NULL, NULL, NULL, NULL,
+                                             xdata);
+                        return 0;
+                }
+        }
+
+simple_unwind:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->rename, oldloc, newloc,
+                         xdata);
+        return 0;
+}
+
+int32_t
+pcli_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+        data_t          *data;
+        uint64_t        value;
+
+        /*
+         * We can't use dict_get_str and strcmp here, because the value comes
+         * directly from the user and might not be NUL-terminated (it would
+         * be if we had set it ourselves.
+         */
+
+        data = dict_get(dict,PROTECT_KEY);
+        if (!data) {
+                goto simple_wind;
+        }
+
+        if (dict->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "attempted to mix %s with other keys", PROTECT_KEY);
+                goto simple_wind;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "got %s request", PROTECT_KEY);
+        if (!strncmp(data->data,"log",data->len)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "logging removals on %s", loc->path);
+                value = PROT_ACT_LOG;
+        }
+        else if (!strncmp(data->data,"reject",data->len)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "rejecting removals on %s", loc->path);
+                value = PROT_ACT_REJECT;
+        }
+        else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "removing protection on %s", loc->path);
+                value = PROT_ACT_NONE;
+        }
+        /* Right now the value doesn't matter - just the presence. */
+        if (inode_ctx_set(loc->inode,this,&value) != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set protection status for %s", loc->path);
+        }
+        STACK_UNWIND_STRICT (setxattr, frame, 0, 0, NULL);
+        return 0;
+
+simple_wind:
+        STACK_WIND_TAIL (frame,
+                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                         loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+pcli_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
+{
+        uint64_t        value;
+
+        if (!loc->parent || (inode_ctx_get(loc->parent,this,&value) != 0)) {
+                goto simple_unwind;
+        }
+
+        if (value != PROT_ACT_NONE) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "got unlink for protected %s", loc->path);
+                pcli_print_trace(this->name,frame->next);
+                if (value == PROT_ACT_REJECT) {
+                        STACK_UNWIND_STRICT (unlink, frame, -1, EPERM,
+                                             NULL, NULL, NULL);
+                        return 0;
+                }
+        }
+
+simple_unwind:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+
+struct xlator_fops fops = {
+        .rename         = pcli_rename,
+        .setxattr       = pcli_setxattr,
+        .unlink         = pcli_unlink,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/protect/src/prot_dht.c b/xlators/features/protect/src/prot_dht.c
new file mode 100644
index 000000000..feec6ffd6
--- /dev/null
+++ b/xlators/features/protect/src/prot_dht.c
@@ -0,0 +1,168 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+enum gf_pdht_mem_types_ {
+        gf_pdht_mt_coord_t = gf_common_mt_end + 1,
+        gf_pdht_mt_end
+};
+
+typedef struct {
+        pthread_mutex_t         lock;
+        uint16_t                refs;
+        int32_t                 op_ret;
+        int32_t                 op_errno;
+        dict_t                  *xdata;
+} pdht_coord_t;
+
+static char PROTECT_KEY[] = "trusted.glusterfs.protect";
+
+void
+pdht_unref_and_unlock (call_frame_t *frame, xlator_t *this,
+                       pdht_coord_t *coord)
+{
+        gf_boolean_t    should_unwind;
+
+        should_unwind = (--(coord->refs) == 0);
+        pthread_mutex_unlock(&coord->lock);
+
+        if (should_unwind) {
+                STACK_UNWIND_STRICT (setxattr, frame, 
+                                     coord->op_ret, coord->op_errno,
+                                     coord->xdata);
+                if (coord->xdata) {
+                        dict_unref(coord->xdata);
+                }
+                GF_FREE(coord);
+        }
+}
+
+int32_t
+pdht_recurse_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        pdht_coord_t    *coord = cookie;
+
+        pthread_mutex_lock(&coord->lock);
+        if (op_ret) {
+                coord->op_ret = op_ret;
+                coord->op_errno = op_errno;
+        }
+        if (xdata) {
+                if (coord->xdata) {
+                        dict_unref(coord->xdata);
+                }
+                coord->xdata = dict_ref(xdata);
+        }
+        pdht_unref_and_unlock(frame,this,coord);
+                
+        return 0;
+}
+
+void
+pdht_recurse (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+              int32_t flags, dict_t *xdata, xlator_t *xl, pdht_coord_t *coord)
+{
+        xlator_list_t   *iter;
+
+        if (!strcmp(xl->type,"features/prot_client")) {
+                pthread_mutex_lock(&coord->lock);
+                ++(coord->refs);
+                pthread_mutex_unlock(&coord->lock);
+                STACK_WIND_COOKIE (frame, pdht_recurse_cbk, coord, xl,
+                                   xl->fops->setxattr, loc, dict, flags, xdata);
+        }
+
+        else for (iter = xl->children; iter; iter = iter->next) {
+                pdht_recurse (frame, this, loc, dict, flags, xdata,
+                              iter->xlator, coord);
+        }
+}
+
+int32_t
+pdht_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+        pdht_coord_t    *coord;
+
+        if (!dict_get(dict,PROTECT_KEY)) {
+                goto simple_wind;
+        }
+
+        if (dict->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "attempted to mix %s with other keys", PROTECT_KEY);
+                goto simple_wind;
+        }
+
+        coord = GF_CALLOC(1,sizeof(*coord),gf_pdht_mt_coord_t);
+        if (!coord) {
+                gf_log (this->name, GF_LOG_WARNING, "allocation failed");
+                goto simple_wind;
+        }
+
+        pthread_mutex_init(&coord->lock,NULL);
+        coord->refs = 1;
+        coord->op_ret = 0;
+        coord->xdata = NULL;
+
+        pdht_recurse(frame,this,loc,dict,flags,xdata,this,coord);
+        pthread_mutex_lock(&coord->lock);
+        pdht_unref_and_unlock(frame,this,coord);
+
+        return 0;
+
+simple_wind:
+        STACK_WIND_TAIL (frame,
+                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                         loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+struct xlator_fops fops = {
+        .setxattr = pdht_setxattr,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/protect/src/prot_server.c b/xlators/features/protect/src/prot_server.c
new file mode 100644
index 000000000..beaee0889
--- /dev/null
+++ b/xlators/features/protect/src/prot_server.c
@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+
+struct xlator_fops fops = {
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/qemu-block/Makefile.am b/xlators/features/qemu-block/Makefile.am
new file mode 100644
index 000000000..af437a64d
--- /dev/null
+++ b/xlators/features/qemu-block/Makefile.am
@@ -0,0 +1 @@
+SUBDIRS = src
diff --git a/xlators/features/qemu-block/src/Makefile.am b/xlators/features/qemu-block/src/Makefile.am
new file mode 100644
index 000000000..08a7b62a0
--- /dev/null
+++ b/xlators/features/qemu-block/src/Makefile.am
@@ -0,0 +1,155 @@
+if ENABLE_QEMU_BLOCK
+xlator_LTLIBRARIES = qemu-block.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+qemu_block_la_LDFLAGS = -module -avoid-version
+qemu_block_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GLIB_LIBS) -lz -lrt
+
+qemu_block_la_SOURCES_qemu = \
+	$(CONTRIBDIR)/qemu/qemu-coroutine.c \
+	$(CONTRIBDIR)/qemu/qemu-coroutine-lock.c \
+	$(CONTRIBDIR)/qemu/qemu-coroutine-sleep.c \
+	$(CONTRIBDIR)/qemu/coroutine-ucontext.c \
+	$(CONTRIBDIR)/qemu/block.c \
+	$(CONTRIBDIR)/qemu/nop-symbols.c
+
+qemu_block_la_SOURCES_qemu_util = \
+	$(CONTRIBDIR)/qemu/util/aes.c \
+	$(CONTRIBDIR)/qemu/util/bitmap.c \
+	$(CONTRIBDIR)/qemu/util/bitops.c \
+	$(CONTRIBDIR)/qemu/util/cutils.c \
+	$(CONTRIBDIR)/qemu/util/error.c \
+	$(CONTRIBDIR)/qemu/util/hbitmap.c \
+	$(CONTRIBDIR)/qemu/util/iov.c \
+	$(CONTRIBDIR)/qemu/util/module.c \
+	$(CONTRIBDIR)/qemu/util/oslib-posix.c \
+	$(CONTRIBDIR)/qemu/util/qemu-option.c \
+	$(CONTRIBDIR)/qemu/util/qemu-error.c \
+	$(CONTRIBDIR)/qemu/util/qemu-thread-posix.c \
+	$(CONTRIBDIR)/qemu/util/unicode.c \
+	$(CONTRIBDIR)/qemu/util/hexdump.c
+
+qemu_block_la_SOURCES_qemu_block = \
+	$(CONTRIBDIR)/qemu/block/snapshot.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-cache.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-cluster.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-refcount.c \
+	$(CONTRIBDIR)/qemu/block/qcow2-snapshot.c \
+	$(CONTRIBDIR)/qemu/block/qcow2.c \
+	$(CONTRIBDIR)/qemu/block/qed-check.c \
+	$(CONTRIBDIR)/qemu/block/qed-cluster.c \
+	$(CONTRIBDIR)/qemu/block/qed-gencb.c \
+	$(CONTRIBDIR)/qemu/block/qed-l2-cache.c \
+	$(CONTRIBDIR)/qemu/block/qed-table.c \
+	$(CONTRIBDIR)/qemu/block/qed.c
+
+qemu_block_la_SOURCES_qemu_qobject = \
+	$(CONTRIBDIR)/qemu/qobject/json-lexer.c \
+	$(CONTRIBDIR)/qemu/qobject/json-parser.c \
+	$(CONTRIBDIR)/qemu/qobject/json-streamer.c \
+	$(CONTRIBDIR)/qemu/qobject/qbool.c \
+	$(CONTRIBDIR)/qemu/qobject/qdict.c \
+	$(CONTRIBDIR)/qemu/qobject/qerror.c \
+	$(CONTRIBDIR)/qemu/qobject/qfloat.c \
+	$(CONTRIBDIR)/qemu/qobject/qint.c \
+	$(CONTRIBDIR)/qemu/qobject/qjson.c \
+	$(CONTRIBDIR)/qemu/qobject/qlist.c \
+	$(CONTRIBDIR)/qemu/qobject/qstring.c
+
+qemu_block_la_SOURCES = \
+	$(qemu_block_la_SOURCES_qemu) \
+	$(qemu_block_la_SOURCES_qemu_util) \
+	$(qemu_block_la_SOURCES_qemu_block) \
+	$(qemu_block_la_SOURCES_qemu_qobject) \
+	bdrv-xlator.c \
+	coroutine-synctask.c \
+	bh-syncop.c \
+	monitor-logging.c \
+	clock-timer.c \
+	qemu-block.c \
+	qb-coroutines.c
+
+noinst_HEADERS_qemu = \
+	$(CONTRIBDIR)/qemu/config-host.h \
+	$(CONTRIBDIR)/qemu/qapi-types.h \
+	$(CONTRIBDIR)/qemu/qmp-commands.h \
+	$(CONTRIBDIR)/qemu/trace/generated-tracers.h \
+	$(CONTRIBDIR)/qemu/include/config.h \
+	$(CONTRIBDIR)/qemu/include/glib-compat.h \
+	$(CONTRIBDIR)/qemu/include/qemu-common.h \
+	$(CONTRIBDIR)/qemu/include/trace.h \
+	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
+	$(CONTRIBDIR)/qemu/include/block/aio.h \
+	$(CONTRIBDIR)/qemu/include/block/block.h \
+	$(CONTRIBDIR)/qemu/include/block/block_int.h \
+	$(CONTRIBDIR)/qemu/include/block/blockjob.h \
+	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
+	$(CONTRIBDIR)/qemu/include/block/coroutine_int.h \
+	$(CONTRIBDIR)/qemu/include/block/snapshot.h \
+	$(CONTRIBDIR)/qemu/include/exec/cpu-common.h \
+	$(CONTRIBDIR)/qemu/include/exec/hwaddr.h \
+	$(CONTRIBDIR)/qemu/include/exec/poison.h \
+	$(CONTRIBDIR)/qemu/include/fpu/softfloat.h \
+	$(CONTRIBDIR)/qemu/include/migration/migration.h \
+	$(CONTRIBDIR)/qemu/include/migration/qemu-file.h \
+	$(CONTRIBDIR)/qemu/include/migration/vmstate.h \
+	$(CONTRIBDIR)/qemu/include/monitor/monitor.h \
+	$(CONTRIBDIR)/qemu/include/monitor/readline.h \
+	$(CONTRIBDIR)/qemu/include/qapi/error.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-lexer.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-parser.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-streamer.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qbool.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qdict.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qerror.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qfloat.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qint.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qjson.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qlist.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qobject.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/qstring.h \
+	$(CONTRIBDIR)/qemu/include/qapi/qmp/types.h \
+	$(CONTRIBDIR)/qemu/include/qemu/aes.h \
+	$(CONTRIBDIR)/qemu/include/qemu/atomic.h \
+	$(CONTRIBDIR)/qemu/include/qemu/bitmap.h \
+	$(CONTRIBDIR)/qemu/include/qemu/bitops.h \
+	$(CONTRIBDIR)/qemu/include/qemu/bswap.h \
+	$(CONTRIBDIR)/qemu/include/qemu/compiler.h \
+	$(CONTRIBDIR)/qemu/include/qemu/error-report.h \
+	$(CONTRIBDIR)/qemu/include/qemu/event_notifier.h \
+	$(CONTRIBDIR)/qemu/include/qemu/hbitmap.h \
+	$(CONTRIBDIR)/qemu/include/qemu/host-utils.h \
+	$(CONTRIBDIR)/qemu/include/qemu/iov.h \
+	$(CONTRIBDIR)/qemu/include/qemu/main-loop.h \
+	$(CONTRIBDIR)/qemu/include/qemu/module.h \
+	$(CONTRIBDIR)/qemu/include/qemu/notify.h \
+	$(CONTRIBDIR)/qemu/include/qemu/option.h \
+	$(CONTRIBDIR)/qemu/include/qemu/option_int.h \
+	$(CONTRIBDIR)/qemu/include/qemu/osdep.h \
+	$(CONTRIBDIR)/qemu/include/qemu/queue.h \
+	$(CONTRIBDIR)/qemu/include/qemu/sockets.h \
+	$(CONTRIBDIR)/qemu/include/qemu/thread-posix.h \
+	$(CONTRIBDIR)/qemu/include/qemu/thread.h \
+	$(CONTRIBDIR)/qemu/include/qemu/timer.h \
+	$(CONTRIBDIR)/qemu/include/qemu/typedefs.h \
+	$(CONTRIBDIR)/qemu/include/sysemu/sysemu.h \
+	$(CONTRIBDIR)/qemu/include/sysemu/os-posix.h \
+	$(CONTRIBDIR)/qemu/block/qcow2.h \
+	$(CONTRIBDIR)/qemu/block/qed.h
+
+noinst_HEADERS = \
+	$(noinst_HEADERS_qemu) \
+	qemu-block.h \
+	qemu-block-memory-types.h \
+	qb-coroutines.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+            -I$(CONTRIBDIR)/qemu \
+            -I$(CONTRIBDIR)/qemu/include \
+            -DGLUSTER_XLATOR
+
+AM_CFLAGS = -fno-strict-aliasing -Wall $(GF_CFLAGS) $(GLIB_CFLAGS)
+
+CLEANFILES =
+
+endif
diff --git a/xlators/features/qemu-block/src/bdrv-xlator.c b/xlators/features/qemu-block/src/bdrv-xlator.c
new file mode 100644
index 000000000..106c59775
--- /dev/null
+++ b/xlators/features/qemu-block/src/bdrv-xlator.c
@@ -0,0 +1,397 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "inode.h"
+#include "syncop.h"
+#include "qemu-block.h"
+#include "block/block_int.h"
+
+typedef struct BDRVGlusterState {
+	inode_t *inode;
+} BDRVGlusterState;
+
+static QemuOptsList runtime_opts = {
+	.name = "gluster",
+	.head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
+	.desc = {
+		{
+			.name = "filename",
+			.type = QEMU_OPT_STRING,
+			.help = "GFID of file",
+		},
+		{ /* end of list */ }
+	},
+};
+
+inode_t *
+qb_inode_from_filename (const char *filename)
+{
+	const char *iptr = NULL;
+	inode_t *inode = NULL;
+
+	iptr = filename + 17;
+	sscanf (iptr, "%p", &inode);
+
+	return inode;
+}
+
+
+int
+qb_inode_to_filename (inode_t *inode, char *filename, int size)
+{
+	return snprintf (filename, size, "gluster://inodep:%p", inode);
+}
+
+
+static fd_t *
+fd_from_bs (BlockDriverState *bs)
+{
+	BDRVGlusterState *s = bs->opaque;
+
+	return fd_anonymous (s->inode);
+}
+
+
+static int
+qemu_gluster_open (BlockDriverState *bs, QDict *options, int bdrv_flags)
+{
+	inode_t *inode = NULL;
+	BDRVGlusterState *s = bs->opaque;
+	QemuOpts *opts = NULL;
+	Error *local_err = NULL;
+	const char *filename = NULL;
+	char gfid_str[128];
+	int ret;
+	qb_conf_t *conf = THIS->private;
+
+	opts = qemu_opts_create_nofail(&runtime_opts);
+	qemu_opts_absorb_qdict(opts, options, &local_err);
+	if (error_is_set(&local_err)) {
+		qerror_report_err(local_err);
+		error_free(local_err);
+		return -EINVAL;
+	}
+
+	filename = qemu_opt_get(opts, "filename");
+
+	/*
+	 * gfid:<gfid> format means we're opening a backing image.
+	 */
+	ret = sscanf(filename, "gluster://gfid:%s", gfid_str);
+	if (ret) {
+		loc_t loc = {0,};
+		struct iatt buf = {0,};
+		uuid_t gfid;
+
+		uuid_parse(gfid_str, gfid);
+
+		loc.inode = inode_find(conf->root_inode->table, gfid);
+		if (!loc.inode) {
+			loc.inode = inode_new(conf->root_inode->table);
+			uuid_copy(loc.inode->gfid, gfid);
+		}
+
+		uuid_copy(loc.gfid, loc.inode->gfid);
+		ret = syncop_lookup(FIRST_CHILD(THIS), &loc, NULL, &buf, NULL,
+				    NULL);
+		if (ret) {
+			loc_wipe(&loc);
+			return -errno;
+		}
+
+		s->inode = inode_ref(loc.inode);
+		loc_wipe(&loc);
+	} else {
+		inode = qb_inode_from_filename (filename);
+		if (!inode)
+			return -EINVAL;
+
+		s->inode = inode_ref(inode);
+	}
+
+	return 0;
+}
+
+
+static int
+qemu_gluster_create (const char *filename, QEMUOptionParameter *options)
+{
+	uint64_t total_size = 0;
+	inode_t *inode = NULL;
+	fd_t *fd = NULL;
+	struct iatt stat = {0, };
+	int ret = 0;
+
+	inode = qb_inode_from_filename (filename);
+	if (!inode)
+		return -EINVAL;
+
+	while (options && options->name) {
+		if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
+			total_size = options->value.n / BDRV_SECTOR_SIZE;
+		}
+		options++;
+	}
+
+	fd = fd_anonymous (inode);
+	if (!fd)
+		return -ENOMEM;
+
+	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &stat);
+	if (ret) {
+		fd_unref (fd);
+		return -errno;
+	}
+
+	if (stat.ia_size) {
+		/* format ONLY if the filesize is 0 bytes */
+		fd_unref (fd);
+		return -EFBIG;
+	}
+
+	if (total_size) {
+		ret = syncop_ftruncate (FIRST_CHILD(THIS), fd, total_size);
+		if (ret) {
+			fd_unref (fd);
+			return -errno;
+		}
+	}
+
+	fd_unref (fd);
+	return 0;
+}
+
+
+static int
+qemu_gluster_co_readv (BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+		       QEMUIOVector *qiov)
+{
+	fd_t *fd = NULL;
+	off_t offset = 0;
+	size_t size = 0;
+	struct iovec *iov = NULL;
+	int count = 0;
+	struct iobref *iobref = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+	if (!fd)
+		return -EIO;
+
+	offset = sector_num * BDRV_SECTOR_SIZE;
+	size = nb_sectors * BDRV_SECTOR_SIZE;
+
+	ret = syncop_readv (FIRST_CHILD(THIS), fd, size, offset, 0,
+			    &iov, &count, &iobref);
+	if (ret < 0) {
+		ret = -errno;
+		goto out;
+	}
+
+	iov_copy (qiov->iov, qiov->niov, iov, count); /* *choke!* */
+
+out:
+	GF_FREE (iov);
+	if (iobref)
+		iobref_unref (iobref);
+	fd_unref (fd);
+	return ret;
+}
+
+
+static int
+qemu_gluster_co_writev (BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+			QEMUIOVector *qiov)
+{
+	fd_t *fd = NULL;
+	off_t offset = 0;
+	size_t size = 0;
+	struct iobref *iobref = NULL;
+	struct iobuf *iobuf = NULL;
+	struct iovec iov = {0, };
+	int ret = -ENOMEM;
+
+	fd = fd_from_bs (bs);
+	if (!fd)
+		return -EIO;
+
+	offset = sector_num * BDRV_SECTOR_SIZE;
+	size = nb_sectors * BDRV_SECTOR_SIZE;
+
+	iobuf = iobuf_get2 (THIS->ctx->iobuf_pool, size);
+	if (!iobuf)
+		goto out;
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		iobuf_unref (iobuf);
+		goto out;
+	}
+
+	iobref_add (iobref, iobuf);
+
+	iov_unload (iobuf_ptr (iobuf), qiov->iov, qiov->niov); /* *choke!* */
+
+	iov.iov_base = iobuf_ptr (iobuf);
+	iov.iov_len = size;
+
+	ret = syncop_writev (FIRST_CHILD(THIS), fd, &iov, 1, offset, iobref, 0);
+	if (ret < 0)
+		ret = -errno;
+
+out:
+	if (iobuf)
+		iobuf_unref (iobuf);
+	if (iobref)
+		iobref_unref (iobref);
+	fd_unref (fd);
+	return ret;
+}
+
+
+static int
+qemu_gluster_co_flush (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_flush (FIRST_CHILD(THIS), fd);
+
+	fd_unref (fd);
+
+	return ret;
+}
+
+
+static int
+qemu_gluster_co_fsync (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_fsync (FIRST_CHILD(THIS), fd, 0);
+
+	fd_unref (fd);
+
+	return ret;
+}
+
+
+static int
+qemu_gluster_truncate (BlockDriverState *bs, int64_t offset)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_ftruncate (FIRST_CHILD(THIS), fd, offset);
+
+	fd_unref (fd);
+
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+
+static int64_t
+qemu_gluster_getlength (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+	struct iatt iatt = {0, };
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &iatt);
+	if (ret < 0)
+		return -1;
+
+	return iatt.ia_size;
+}
+
+
+static int64_t
+qemu_gluster_allocated_file_size (BlockDriverState *bs)
+{
+	fd_t *fd = NULL;
+	int ret = 0;
+	struct iatt iatt = {0, };
+
+	fd = fd_from_bs (bs);
+
+	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &iatt);
+	if (ret < 0)
+		return -1;
+
+	return iatt.ia_blocks * 512;
+}
+
+
+static void
+qemu_gluster_close (BlockDriverState *bs)
+{
+	BDRVGlusterState *s = NULL;
+
+	s = bs->opaque;
+
+	inode_unref (s->inode);
+
+	return;
+}
+
+
+static QEMUOptionParameter qemu_gluster_create_options[] = {
+	{
+		.name = BLOCK_OPT_SIZE,
+		.type = OPT_SIZE,
+		.help = "Virtual disk size"
+	},
+	{ NULL }
+};
+
+
+static BlockDriver bdrv_gluster = {
+	.format_name                  = "gluster",
+	.protocol_name                = "gluster",
+	.instance_size                = sizeof(BDRVGlusterState),
+	.bdrv_file_open               = qemu_gluster_open,
+	.bdrv_close                   = qemu_gluster_close,
+	.bdrv_create                  = qemu_gluster_create,
+	.bdrv_getlength               = qemu_gluster_getlength,
+	.bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+	.bdrv_co_readv                = qemu_gluster_co_readv,
+	.bdrv_co_writev               = qemu_gluster_co_writev,
+	.bdrv_co_flush_to_os          = qemu_gluster_co_flush,
+	.bdrv_co_flush_to_disk        = qemu_gluster_co_fsync,
+	.bdrv_truncate                = qemu_gluster_truncate,
+	.create_options               = qemu_gluster_create_options,
+};
+
+
+static void bdrv_gluster_init(void)
+{
+	bdrv_register(&bdrv_gluster);
+}
+
+
+block_init(bdrv_gluster_init);
diff --git a/xlators/features/qemu-block/src/bh-syncop.c b/xlators/features/qemu-block/src/bh-syncop.c
new file mode 100644
index 000000000..e8686f6d4
--- /dev/null
+++ b/xlators/features/qemu-block/src/bh-syncop.c
@@ -0,0 +1,48 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "syncop.h"
+#include "qemu-block-memory-types.h"
+
+#include "block/aio.h"
+
+void
+qemu_bh_schedule (QEMUBH *bh)
+{
+	return;
+}
+
+void
+qemu_bh_cancel (QEMUBH *bh)
+{
+	return;
+}
+
+void
+qemu_bh_delete (QEMUBH *bh)
+{
+
+}
+
+QEMUBH *
+qemu_bh_new (QEMUBHFunc *cb, void *opaque)
+{
+	return NULL;
+}
diff --git a/xlators/features/qemu-block/src/clock-timer.c b/xlators/features/qemu-block/src/clock-timer.c
new file mode 100644
index 000000000..fcbec6ad1
--- /dev/null
+++ b/xlators/features/qemu-block/src/clock-timer.c
@@ -0,0 +1,60 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "syncop.h"
+#include "qemu-block-memory-types.h"
+
+#include "qemu/timer.h"
+
+QEMUClock *vm_clock;
+int use_rt_clock = 0;
+
+QEMUTimer *qemu_new_timer (QEMUClock *clock, int scale,
+			   QEMUTimerCB *cb, void *opaque)
+{
+	return NULL;
+}
+
+int64_t qemu_get_clock_ns (QEMUClock *clock)
+{
+	return 0;
+}
+
+void qemu_mod_timer (QEMUTimer *ts, int64_t expire_time)
+{
+	return;
+}
+
+void qemu_free_timer (QEMUTimer *ts)
+{
+
+}
+
+void qemu_del_timer (QEMUTimer *ts)
+{
+
+}
+
+bool qemu_aio_wait()
+{
+	synctask_wake (synctask_get());
+	synctask_yield (synctask_get());
+	return 0;
+}
diff --git a/xlators/features/qemu-block/src/coroutine-synctask.c b/xlators/features/qemu-block/src/coroutine-synctask.c
new file mode 100644
index 000000000..e43988a95
--- /dev/null
+++ b/xlators/features/qemu-block/src/coroutine-synctask.c
@@ -0,0 +1,116 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "syncop.h"
+#include "qemu-block-memory-types.h"
+
+#include "qemu-block.h"
+
+/*
+ * This code serves as the bridge from the main glusterfs context to the qemu
+ * coroutine context via synctask. We create a single threaded syncenv with a
+ * single synctask responsible for processing a queue of coroutines. The qemu
+ * code invoked from within the synctask function handlers uses the ucontext
+ * coroutine implementation and scheduling logic internal to qemu. This
+ * effectively donates a thread of execution to qemu and its internal coroutine
+ * management.
+ *
+ * NOTE: The existence of concurrent synctasks has proven quite racy with regard
+ * to qemu coroutine management, particularly related to the lifecycle
+ * differences with top-level synctasks and internally created coroutines and
+ * interactions with qemu-internal queues (and locks, in turn). We explicitly
+ * disallow this scenario, via the queue, until it is more well supported.
+ */
+
+static struct {
+	struct list_head	queue;
+	gf_lock_t		lock;
+	struct synctask		*task;
+} qb_co;
+
+static void
+init_qbco()
+{
+	INIT_LIST_HEAD(&qb_co.queue);
+	LOCK_INIT(&qb_co.lock);
+}
+
+static int
+synctask_nop_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+	return 0;
+}
+
+static int
+qb_synctask_wrap (void *opaque)
+{
+	qb_local_t *qb_local, *tmp;
+
+	LOCK(&qb_co.lock);
+
+	while (!list_empty(&qb_co.queue)) {
+		list_for_each_entry_safe(qb_local, tmp, &qb_co.queue, list) {
+			list_del_init(&qb_local->list);
+			break;
+		}
+
+		UNLOCK(&qb_co.lock);
+
+		qb_local->synctask_fn(qb_local);
+		/* qb_local is now unwound and gone! */
+
+		LOCK(&qb_co.lock);
+	}
+
+	qb_co.task = NULL;
+
+	UNLOCK(&qb_co.lock);
+
+	return 0;
+}
+
+int
+qb_coroutine (call_frame_t *frame, synctask_fn_t fn)
+{
+	qb_local_t *qb_local = NULL;
+	qb_conf_t *qb_conf = NULL;
+	static int init = 0;
+
+	qb_local = frame->local;
+	qb_local->synctask_fn = fn;
+	qb_conf = frame->this->private;
+
+	if (!init) {
+		init = 1;
+		init_qbco();
+	}
+
+	LOCK(&qb_co.lock);
+
+	if (!qb_co.task)
+		qb_co.task = synctask_create(qb_conf->env, qb_synctask_wrap,
+					     synctask_nop_cbk, frame, NULL);
+
+	list_add_tail(&qb_local->list, &qb_co.queue);
+
+	UNLOCK(&qb_co.lock);
+
+	return 0;
+}
diff --git a/xlators/features/qemu-block/src/monitor-logging.c b/xlators/features/qemu-block/src/monitor-logging.c
new file mode 100644
index 000000000..d37c37f0f
--- /dev/null
+++ b/xlators/features/qemu-block/src/monitor-logging.c
@@ -0,0 +1,50 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "qemu-block-memory-types.h"
+
+#include "block/block_int.h"
+
+Monitor *cur_mon;
+
+int
+monitor_cur_is_qmp()
+{
+	/* No QMP support here */
+	return 0;
+}
+
+void
+monitor_set_error (Monitor *mon, QError *qerror)
+{
+	/* NOP here */
+	return;
+}
+
+
+void
+monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
+{
+	char buf[4096];
+
+	vsnprintf(buf, sizeof(buf), fmt, ap);
+
+	gf_log (THIS->name, GF_LOG_ERROR, "%s", buf);
+}
diff --git a/xlators/features/qemu-block/src/qb-coroutines.c b/xlators/features/qemu-block/src/qb-coroutines.c
new file mode 100644
index 000000000..7c52adb21
--- /dev/null
+++ b/xlators/features/qemu-block/src/qb-coroutines.c
@@ -0,0 +1,662 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "inode.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "qemu-block-memory-types.h"
+#include "qemu-block.h"
+#include "qb-coroutines.h"
+
+
+int
+qb_format_and_resume (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	char filename[64];
+	char base_filename[128];
+	int use_base = 0;
+	qb_inode_t *qb_inode = NULL;
+	Error *local_err = NULL;
+	fd_t *fd = NULL;
+	dict_t *xattr = NULL;
+	qb_conf_t *qb_conf = NULL;
+	int ret = -1;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+	qb_conf = frame->this->private;
+
+	qb_inode_to_filename (inode, filename, 64);
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+
+	/*
+	 * See if the caller specified a backing image.
+	 */
+	if (!uuid_is_null(qb_inode->backing_gfid) || qb_inode->backing_fname) {
+		loc_t loc = {0,};
+		char gfid_str[64];
+		struct iatt buf;
+
+		if (!uuid_is_null(qb_inode->backing_gfid)) {
+			loc.inode = inode_find(qb_conf->root_inode->table,
+					qb_inode->backing_gfid);
+			if (!loc.inode) {
+				loc.inode = inode_new(qb_conf->root_inode->table);
+				uuid_copy(loc.inode->gfid,
+					qb_inode->backing_gfid);
+			}
+			uuid_copy(loc.gfid, loc.inode->gfid);
+		} else if (qb_inode->backing_fname) {
+			loc.inode = inode_new(qb_conf->root_inode->table);
+			loc.name = qb_inode->backing_fname;
+			loc.parent = inode_parent(inode, NULL, NULL);
+			loc_path(&loc, loc.name);
+		}
+
+		/*
+		 * Lookup the backing image. Verify existence and/or get the
+		 * gfid if we don't already have it.
+		 */
+		ret = syncop_lookup(FIRST_CHILD(frame->this), &loc, NULL, &buf,
+				    NULL, NULL);
+		GF_FREE(qb_inode->backing_fname);
+		if (ret) {
+			loc_wipe(&loc);
+			ret = errno;
+			goto err;
+		}
+
+		uuid_copy(qb_inode->backing_gfid, buf.ia_gfid);
+		loc_wipe(&loc);
+
+		/*
+		 * We pass the filename of the backing image into the qemu block
+		 * subsystem as the associated gfid. This is embedded into the
+		 * clone image and passed along to the gluster bdrv backend when
+		 * the block subsystem needs to operate on the backing image on
+		 * behalf of the clone.
+		 */
+		uuid_unparse(qb_inode->backing_gfid, gfid_str);
+		snprintf(base_filename, sizeof(base_filename),
+			 "gluster://gfid:%s", gfid_str);
+		use_base = 1;
+	}
+
+	bdrv_img_create (filename, qb_inode->fmt,
+			 use_base ? base_filename : NULL, 0, 0, qb_inode->size,
+			 0, &local_err, true);
+
+	if (error_is_set (&local_err)) {
+		gf_log (frame->this->name, GF_LOG_ERROR, "%s",
+			error_get_pretty (local_err));
+		error_free (local_err);
+		QB_STUB_UNWIND (stub, -1, EIO);
+		return 0;
+	}
+
+	fd = fd_anonymous (inode);
+	if (!fd) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"could not create anonymous fd for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		return 0;
+	}
+
+	xattr = dict_new ();
+	if (!xattr) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"could not allocate xattr dict for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		fd_unref (fd);
+		return 0;
+	}
+
+	ret = dict_set_str (xattr, qb_conf->qb_xattr_key, local->fmt);
+	if (ret) {
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"could not dict_set for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		fd_unref (fd);
+		dict_unref (xattr);
+		return 0;
+	}
+
+	ret = syncop_fsetxattr (FIRST_CHILD(THIS), fd, xattr, 0);
+	if (ret) {
+		ret = errno;
+		gf_log (frame->this->name, GF_LOG_ERROR,
+			"failed to setxattr for %s",
+			uuid_utoa (inode->gfid));
+		QB_STUB_UNWIND (stub, -1, ret);
+		fd_unref (fd);
+		dict_unref (xattr);
+		return 0;
+	}
+
+	fd_unref (fd);
+	dict_unref (xattr);
+
+	QB_STUB_UNWIND (stub, 0, 0);
+
+	return 0;
+
+err:
+	QB_STUB_UNWIND(stub, -1, ret);
+	return 0;
+}
+
+
+static BlockDriverState *
+qb_bs_create (inode_t *inode, const char *fmt)
+{
+	char filename[64];
+	BlockDriverState *bs = NULL;
+	BlockDriver *drv = NULL;
+	int op_errno = 0;
+	int ret = 0;
+
+	bs = bdrv_new (uuid_utoa (inode->gfid));
+	if (!bs) {
+		op_errno = ENOMEM;
+		gf_log (THIS->name, GF_LOG_ERROR,
+			"could not allocate @bdrv for gfid:%s",
+			uuid_utoa (inode->gfid));
+		goto err;
+	}
+
+	drv = bdrv_find_format (fmt);
+	if (!drv) {
+		op_errno = EINVAL;
+		gf_log (THIS->name, GF_LOG_ERROR,
+			"Unknown file format: %s for gfid:%s",
+			fmt, uuid_utoa (inode->gfid));
+		goto err;
+	}
+
+	qb_inode_to_filename (inode, filename, 64);
+
+	ret = bdrv_open (bs, filename, NULL, BDRV_O_RDWR, drv);
+	if (ret < 0) {
+		op_errno = -ret;
+		gf_log (THIS->name, GF_LOG_ERROR,
+			"Unable to bdrv_open() gfid:%s (%s)",
+			uuid_utoa (inode->gfid), strerror (op_errno));
+		goto err;
+	}
+
+	return bs;
+err:
+	errno = op_errno;
+	return NULL;
+}
+
+
+int
+qb_co_open (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+	qb_inode->refcnt++;
+
+	QB_STUB_RESUME (stub);
+
+	return 0;
+}
+
+
+int
+qb_co_writev (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	QEMUIOVector qiov = {0, };
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	qemu_iovec_init_external (&qiov, stub->args.vector, stub->args.count);
+
+	ret = bdrv_pwritev (qb_inode->bs, stub->args.offset, &qiov);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_co_readv (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	struct iobuf *iobuf = NULL;
+	struct iobref *iobref = NULL;
+	struct iovec iov = {0, };
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	if (stub->args.offset >= qb_inode->size) {
+		QB_STUB_UNWIND (stub, 0, 0);
+		return 0;
+	}
+
+	iobuf = iobuf_get2 (frame->this->ctx->iobuf_pool, stub->args.size);
+	if (!iobuf) {
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		return 0;
+	}
+
+	iobref = iobref_new ();
+	if (!iobref) {
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		iobuf_unref (iobuf);
+		return 0;
+	}
+
+	if (iobref_add (iobref, iobuf) < 0) {
+		iobuf_unref (iobuf);
+		iobref_unref (iobref);
+		QB_STUB_UNWIND (stub, -1, ENOMEM);
+		return 0;
+	}
+
+	ret = bdrv_pread (qb_inode->bs, stub->args.offset, iobuf_ptr (iobuf),
+			  stub->args.size);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+		iobref_unref (iobref);
+		return 0;
+	}
+
+	iov.iov_base = iobuf_ptr (iobuf);
+	iov.iov_len = ret;
+
+	stub->args_cbk.vector = iov_dup (&iov, 1);
+	stub->args_cbk.count = 1;
+	stub->args_cbk.iobref = iobref;
+
+	QB_STUB_UNWIND (stub, ret, 0);
+
+	return 0;
+}
+
+
+int
+qb_co_fsync (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = bdrv_flush (qb_inode->bs);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+static void
+qb_update_size_xattr (xlator_t *this, fd_t *fd, const char *fmt, off_t offset)
+{
+	char val[QB_XATTR_VAL_MAX];
+	qb_conf_t *qb_conf = NULL;
+	dict_t *xattr = NULL;
+
+	qb_conf = this->private;
+
+	snprintf (val, QB_XATTR_VAL_MAX, "%s:%llu",
+		  fmt, (long long unsigned) offset);
+
+	xattr = dict_new ();
+	if (!xattr)
+		return;
+
+	if (dict_set_str (xattr, qb_conf->qb_xattr_key, val) != 0) {
+		dict_unref (xattr);
+		return;
+	}
+
+	syncop_fsetxattr (FIRST_CHILD(this), fd, xattr, 0);
+	dict_unref (xattr);
+}
+
+
+int
+qb_co_truncate (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+	off_t offset = 0;
+	xlator_t *this = NULL;
+
+	this = THIS;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	syncop_fstat (FIRST_CHILD(this), local->fd, &stub->args_cbk.prestat);
+	stub->args_cbk.prestat.ia_size = qb_inode->size;
+
+	ret = bdrv_truncate (qb_inode->bs, stub->args.offset);
+	if (ret < 0)
+		goto out;
+
+	offset = bdrv_getlength (qb_inode->bs);
+
+	qb_inode->size = offset;
+
+	syncop_fstat (FIRST_CHILD(this), local->fd, &stub->args_cbk.poststat);
+	stub->args_cbk.poststat.ia_size = qb_inode->size;
+
+	qb_update_size_xattr (this, local->fd, qb_inode->fmt, qb_inode->size);
+
+out:
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_co_close (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	BlockDriverState *bs = NULL;
+
+	local = opaque;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (THIS, inode);
+
+	if (!--qb_inode->refcnt) {
+		bs = qb_inode->bs;
+		qb_inode->bs = NULL;
+		bdrv_delete (bs);
+	}
+
+	frame = local->frame;
+	frame->local = NULL;
+	qb_local_free (THIS, local);
+	STACK_DESTROY (frame->root);
+
+	return 0;
+}
+
+
+int
+qb_snapshot_create (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	QEMUSnapshotInfo sn;
+	struct timeval tv = {0, };
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	memset (&sn, 0, sizeof (sn));
+        pstrcpy (sn.name, sizeof(sn.name), local->name);
+        gettimeofday (&tv, NULL);
+        sn.date_sec = tv.tv_sec;
+        sn.date_nsec = tv.tv_usec * 1000;
+
+        ret = bdrv_snapshot_create (qb_inode->bs, &sn);
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_snapshot_delete (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = bdrv_snapshot_delete (qb_inode->bs, local->name);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
+
+
+int
+qb_snapshot_goto (void *opaque)
+{
+	qb_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	call_stub_t *stub = NULL;
+	inode_t *inode = NULL;
+	qb_inode_t *qb_inode = NULL;
+	int ret = 0;
+
+	local = opaque;
+	frame = local->frame;
+	stub = local->stub;
+	inode = local->inode;
+
+	qb_inode = qb_inode_ctx_get (frame->this, inode);
+	if (!qb_inode->bs) {
+		/* FIXME: we need locks around this when
+		   enabling multithreaded syncop/coroutine
+		   for qemu-block
+		*/
+
+		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
+		if (!qb_inode->bs) {
+			QB_STUB_UNWIND (stub, -1, errno);
+			return 0;
+		}
+	}
+
+	ret = bdrv_snapshot_goto (qb_inode->bs, local->name);
+
+	if (ret < 0) {
+		QB_STUB_UNWIND (stub, -1, -ret);
+	} else {
+		QB_STUB_UNWIND (stub, ret, 0);
+	}
+
+	return 0;
+}
diff --git a/xlators/features/qemu-block/src/qb-coroutines.h b/xlators/features/qemu-block/src/qb-coroutines.h
new file mode 100644
index 000000000..583319f3b
--- /dev/null
+++ b/xlators/features/qemu-block/src/qb-coroutines.h
@@ -0,0 +1,30 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QB_COROUTINES_H
+#define __QB_COROUTINES_H
+
+#include "syncop.h"
+#include "call-stub.h"
+#include "block/block_int.h"
+#include "monitor/monitor.h"
+
+int qb_format_and_resume (void *opaque);
+int qb_snapshot_create (void *opaque);
+int qb_snapshot_delete (void *opaque);
+int qb_snapshot_goto (void *opaque);
+int qb_co_open (void *opaque);
+int qb_co_close (void *opaque);
+int qb_co_writev (void *opaque);
+int qb_co_readv (void *opaque);
+int qb_co_fsync (void *opaque);
+int qb_co_truncate (void *opaque);
+
+#endif /* __QB_COROUTINES_H */
diff --git a/xlators/features/qemu-block/src/qemu-block-memory-types.h b/xlators/features/qemu-block/src/qemu-block-memory-types.h
new file mode 100644
index 000000000..267b3893f
--- /dev/null
+++ b/xlators/features/qemu-block/src/qemu-block-memory-types.h
@@ -0,0 +1,25 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __QB_MEM_TYPES_H__
+#define __QB_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_qb_mem_types_ {
+        gf_qb_mt_qb_conf_t   = gf_common_mt_end + 1,
+        gf_qb_mt_qb_inode_t,
+        gf_qb_mt_qb_local_t,
+	gf_qb_mt_coroutinesynctask_t,
+        gf_qb_mt_end
+};
+#endif
+
diff --git a/xlators/features/qemu-block/src/qemu-block.c b/xlators/features/qemu-block/src/qemu-block.c
new file mode 100644
index 000000000..48bbf3140
--- /dev/null
+++ b/xlators/features/qemu-block/src/qemu-block.c
@@ -0,0 +1,1140 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "inode.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "qemu-block-memory-types.h"
+#include "qemu-block.h"
+#include "qb-coroutines.h"
+
+
+qb_inode_t *
+__qb_inode_ctx_get (xlator_t *this, inode_t *inode)
+{
+        uint64_t    value    = 0;
+        qb_inode_t *qb_inode = NULL;
+
+        __inode_ctx_get (inode, this, &value);
+        qb_inode = (qb_inode_t *)(unsigned long) value;
+
+        return qb_inode;
+}
+
+
+qb_inode_t *
+qb_inode_ctx_get (xlator_t *this, inode_t *inode)
+{
+        qb_inode_t *qb_inode = NULL;
+
+        LOCK (&inode->lock);
+        {
+                qb_inode = __qb_inode_ctx_get (this, inode);
+        }
+        UNLOCK (&inode->lock);
+
+        return qb_inode;
+}
+
+
+qb_inode_t *
+qb_inode_ctx_del (xlator_t *this, inode_t *inode)
+{
+        uint64_t    value    = 0;
+        qb_inode_t *qb_inode = NULL;
+
+        inode_ctx_del (inode, this, &value);
+        qb_inode = (qb_inode_t *)(unsigned long) value;
+
+        return qb_inode;
+}
+
+
+int
+qb_inode_cleanup (xlator_t *this, inode_t *inode, int warn)
+{
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_del (this, inode);
+
+	if (!qb_inode)
+		return 0;
+
+	if (warn)
+		gf_log (this->name, GF_LOG_WARNING,
+			"inode %s no longer block formatted",
+			uuid_utoa (inode->gfid));
+
+	/* free (qb_inode->bs); */
+
+	GF_FREE (qb_inode);
+
+	return 0;
+}
+
+
+int
+qb_iatt_fixup (xlator_t *this, inode_t *inode, struct iatt *iatt)
+{
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, inode);
+	if (!qb_inode)
+		return 0;
+
+	iatt->ia_size = qb_inode->size;
+
+	return 0;
+}
+
+
+int
+qb_format_extract (xlator_t *this, char *format, inode_t *inode)
+{
+	char       *s, *save;
+	uint64_t    size = 0;
+	char        fmt[QB_XATTR_VAL_MAX+1] = {0, };
+	qb_inode_t *qb_inode = NULL;
+	char *formatstr = NULL;
+	uuid_t gfid = {0,};
+	char gfid_str[64] = {0,};
+	int ret;
+
+	strncpy(fmt, format, QB_XATTR_VAL_MAX);
+
+	s = strtok_r(fmt, ":", &save);
+	if (!s)
+		goto invalid;
+	formatstr = gf_strdup(s);
+
+	s = strtok_r(NULL, ":", &save);
+	if (!s)
+		goto invalid;
+	if (gf_string2bytesize (s, &size))
+		goto invalid;
+	if (!size)
+		goto invalid;
+
+	s = strtok_r(NULL, "\0", &save);
+	if (s && !strncmp(s, "<gfid:", strlen("<gfid:"))) {
+		/*
+		 * Check for valid gfid backing image specifier.
+		 */
+		if (strlen(s) + 1 > sizeof(gfid_str))
+			goto invalid;
+		ret = sscanf(s, "<gfid:%[^>]s", gfid_str);
+		if (ret == 1) {
+			ret = uuid_parse(gfid_str, gfid);
+			if (ret < 0)
+				goto invalid;
+		}
+	}
+
+	qb_inode = qb_inode_ctx_get (this, inode);
+	if (!qb_inode)
+		qb_inode = GF_CALLOC (1, sizeof (*qb_inode),
+				      gf_qb_mt_qb_inode_t);
+	if (!qb_inode) {
+		GF_FREE(formatstr);
+		return ENOMEM;
+	}
+
+	strncpy(qb_inode->fmt, formatstr, QB_XATTR_VAL_MAX);
+	qb_inode->size = size;
+
+	/*
+	 * If a backing gfid was not specified, interpret any remaining bytes
+	 * associated with a backing image as a filename local to the parent
+	 * directory. The format processing will validate further.
+	 */
+	if (!uuid_is_null(gfid))
+		uuid_copy(qb_inode->backing_gfid, gfid);
+	else if (s)
+		qb_inode->backing_fname = gf_strdup(s);
+
+	inode_ctx_set (inode, this, (void *)&qb_inode);
+
+	GF_FREE(formatstr);
+
+	return 0;
+
+invalid:
+	GF_FREE(formatstr);
+
+	gf_log (this->name, GF_LOG_WARNING,
+		"invalid format '%s' in inode %s", format,
+		uuid_utoa (inode->gfid));
+	return EINVAL;
+}
+
+
+void
+qb_local_free (xlator_t *this, qb_local_t *local)
+{
+	if (local->inode)
+		inode_unref (local->inode);
+	if (local->fd)
+		fd_unref (local->fd);
+	GF_FREE (local);
+}
+
+
+int
+qb_local_init (call_frame_t *frame)
+{
+	qb_local_t *qb_local = NULL;
+
+	qb_local = GF_CALLOC (1, sizeof (*qb_local), gf_qb_mt_qb_local_t);
+	if (!qb_local)
+		return -1;
+	INIT_LIST_HEAD(&qb_local->list);
+
+	qb_local->frame = frame;
+	frame->local = qb_local;
+
+	return 0;
+}
+
+
+int
+qb_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	       int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+	       dict_t *xdata, struct iatt *postparent)
+{
+	char *format = NULL;
+	qb_conf_t *conf = NULL;
+
+	conf = this->private;
+
+	if (op_ret == -1)
+		goto out;
+
+	/*
+	 * Cache the root inode for dealing with backing images. The format
+	 * coroutine and the gluster qemu backend driver both use the root inode
+	 * table to verify and/or redirect I/O to the backing image via
+	 * anonymous fd's.
+	 */
+	if (!conf->root_inode && __is_root_gfid(inode->gfid))
+		conf->root_inode = inode_ref(inode);
+
+	if (!xdata)
+		goto out;
+
+	if (dict_get_str (xdata, conf->qb_xattr_key, &format))
+		goto out;
+
+	if (!format) {
+		qb_inode_cleanup (this, inode, 1);
+		goto out;
+	}
+
+	op_errno = qb_format_extract (this, format, inode);
+	if (op_errno)
+		op_ret = -1;
+
+	qb_iatt_fixup (this, inode, buf);
+out:
+	QB_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+			 xdata, postparent);
+	return 0;
+}
+
+
+int
+qb_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+	qb_conf_t *conf = NULL;
+
+	conf = this->private;
+
+	xdata = xdata ? dict_ref (xdata) : dict_new ();
+
+	if (!xdata)
+		goto enomem;
+
+	if (dict_set_int32 (xdata, conf->qb_xattr_key, 0))
+		goto enomem;
+
+	STACK_WIND (frame, qb_lookup_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+	dict_unref (xdata);
+	return 0;
+enomem:
+	QB_STACK_UNWIND (lookup, frame, -1, ENOMEM, 0, 0, 0, 0);
+	if (xdata)
+		dict_unref (xdata);
+	return 0;
+}
+
+
+int
+qb_setxattr_format (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
+		    dict_t *xattr, inode_t *inode)
+{
+	char *format = NULL;
+	int op_errno = 0;
+	qb_local_t *qb_local = NULL;
+	data_t *data = NULL;
+	qb_inode_t *qb_inode;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-format"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	format = alloca (data->len + 1);
+	memcpy (format, data->data, data->len);
+	format[data->len] = 0;
+
+	op_errno = qb_format_extract (this, format, inode);
+	if (op_errno) {
+		QB_STUB_UNWIND (stub, -1, op_errno);
+		return 0;
+	}
+	qb_inode = qb_inode_ctx_get(this, inode);
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+
+	snprintf(qb_local->fmt, QB_XATTR_VAL_MAX, "%s:%lu", qb_inode->fmt,
+		 qb_inode->size);
+
+	qb_coroutine (frame, qb_format_and_resume);
+
+	return 0;
+}
+
+
+int
+qb_setxattr_snapshot_create (call_frame_t *frame, xlator_t *this,
+			     call_stub_t *stub, dict_t *xattr, inode_t *inode)
+{
+	qb_local_t *qb_local = NULL;
+	char *name = NULL;
+	data_t *data = NULL;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-create"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	name = alloca (data->len + 1);
+	memcpy (name, data->data, data->len);
+	name[data->len] = 0;
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+	strncpy (qb_local->name, name, 128);
+
+	qb_coroutine (frame, qb_snapshot_create);
+
+	return 0;
+}
+
+
+int
+qb_setxattr_snapshot_delete (call_frame_t *frame, xlator_t *this,
+			     call_stub_t *stub, dict_t *xattr, inode_t *inode)
+{
+	qb_local_t *qb_local = NULL;
+	char *name = NULL;
+	data_t *data = NULL;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-delete"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	name = alloca (data->len + 1);
+	memcpy (name, data->data, data->len);
+	name[data->len] = 0;
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+	strncpy (qb_local->name, name, 128);
+
+	qb_coroutine (frame, qb_snapshot_delete);
+
+	return 0;
+}
+
+int
+qb_setxattr_snapshot_goto (call_frame_t *frame, xlator_t *this,
+			   call_stub_t *stub, dict_t *xattr, inode_t *inode)
+{
+	qb_local_t *qb_local = NULL;
+	char *name = NULL;
+	data_t *data = NULL;
+
+	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-goto"))) {
+		QB_STUB_RESUME (stub);
+		return 0;
+	}
+
+	name = alloca (data->len + 1);
+	memcpy (name, data->data, data->len);
+	name[data->len] = 0;
+
+	qb_local = frame->local;
+
+	qb_local->stub = stub;
+	qb_local->inode = inode_ref (inode);
+	strncpy (qb_local->name, name, 128);
+
+	qb_coroutine (frame, qb_snapshot_goto);
+
+	return 0;
+}
+
+
+int
+qb_setxattr_common (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
+		    dict_t *xattr, inode_t *inode)
+{
+	data_t *data = NULL;
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-format"))) {
+		qb_setxattr_format (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-snapshot-create"))) {
+		qb_setxattr_snapshot_create (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-snapshot-delete"))) {
+		qb_setxattr_snapshot_delete (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	if ((data = dict_get (xattr, "trusted.glusterfs.block-snapshot-goto"))) {
+		qb_setxattr_snapshot_goto (frame, this, stub, xattr, inode);
+		return 0;
+	}
+
+	QB_STUB_RESUME (stub);
+
+	return 0;
+}
+
+
+int
+qb_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr,
+	     int flags, dict_t *xdata)
+{
+	call_stub_t *stub = NULL;
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	stub = fop_setxattr_stub (frame, default_setxattr_resume, loc, xattr,
+				  flags, xdata);
+	if (!stub)
+		goto enomem;
+
+	qb_setxattr_common (frame, this, stub, xattr, loc->inode);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (setxattr, frame, -1, ENOMEM, 0);
+	return 0;
+}
+
+
+int
+qb_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+	      int flags, dict_t *xdata)
+{
+	call_stub_t *stub = NULL;
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume, fd, xattr,
+				   flags, xdata);
+	if (!stub)
+		goto enomem;
+
+	qb_setxattr_common (frame, this, stub, xattr, fd->inode);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (fsetxattr, frame, -1, ENOMEM, 0);
+	return 0;
+}
+
+
+int
+qb_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	     int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+	call_stub_t *stub = NULL;
+	qb_local_t *qb_local = NULL;
+
+	qb_local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+
+	if (!qb_inode_ctx_get (this, qb_local->inode))
+		goto unwind;
+
+	stub = fop_open_cbk_stub (frame, NULL, op_ret, op_errno, fd, xdata);
+	if (!stub) {
+		op_ret = -1;
+		op_errno = ENOMEM;
+		goto unwind;
+	}
+
+	qb_local->stub = stub;
+
+	qb_coroutine (frame, qb_co_open);
+
+	return 0;
+unwind:
+	QB_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
+	return 0;
+}
+
+
+int
+qb_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	 fd_t *fd, dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, loc->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_open_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->open, loc, flags, fd,
+			    xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (loc->inode);
+	qb_local->fd = fd_ref (fd);
+
+	STACK_WIND (frame, qb_open_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+	return 0;
+enomem:
+	QB_STACK_UNWIND (open, frame, -1, ENOMEM, 0, 0);
+	return 0;
+}
+
+
+int
+qb_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
+	   dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_writev_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->writev, fd, vector, count,
+			    offset, flags, iobref, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_writev_stub (frame, NULL, fd, vector, count,
+					  offset, flags, iobref, xdata);
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_writev);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (writev, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	  off_t offset, uint32_t flags, dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->readv, fd, size, offset,
+			    flags, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_readv_stub (frame, NULL, fd, size, offset,
+					  flags, xdata);
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_readv);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (readv, frame, -1, ENOMEM, 0, 0, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int dsync,
+	  dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->fsync, fd, dsync, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_fsync_stub (frame, NULL, fd, dsync, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_fsync);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->flush, fd, xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_flush_stub (frame, NULL, fd, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_fsync);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (flush, frame, -1, ENOMEM, 0);
+	return 0;
+}
+
+static int32_t
+qb_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+		dict_t *xdata)
+{
+	qb_conf_t *conf = this->private;
+	gf_dirent_t *entry;
+	char *format;
+
+	list_for_each_entry(entry, &entries->list, list) {
+		if (!entry->inode || !entry->dict)
+			continue;
+
+		format = NULL;
+		if (dict_get_str(entry->dict, conf->qb_xattr_key, &format))
+			continue;
+
+		if (!format) {
+			qb_inode_cleanup(this, entry->inode, 1);
+			continue;
+		}
+
+		if (qb_format_extract(this, format, entry->inode))
+			continue;
+
+		qb_iatt_fixup(this, entry->inode, &entry->d_stat);
+	}
+
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+static int32_t
+qb_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	    off_t off, dict_t *xdata)
+{
+	qb_conf_t *conf = this->private;
+
+	xdata = xdata ? dict_ref(xdata) : dict_new();
+	if (!xdata)
+		goto enomem;
+
+	if (dict_set_int32 (xdata, conf->qb_xattr_key, 0))
+		goto enomem;
+
+	STACK_WIND(frame, qb_readdirp_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+	dict_unref(xdata);
+	return 0;
+
+enomem:
+	QB_STACK_UNWIND(readdirp, frame, -1, ENOMEM, NULL, NULL);
+	if (xdata)
+		dict_unref(xdata);
+	return 0;
+}
+
+int
+qb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+	     dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, loc->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->truncate, loc, offset,
+			    xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (loc->inode);
+	qb_local->fd = fd_anonymous (loc->inode);
+
+	qb_local->stub = fop_truncate_stub (frame, NULL, loc, offset, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_truncate);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (truncate, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	     dict_t *xdata)
+{
+	qb_local_t *qb_local = NULL;
+	qb_inode_t *qb_inode = NULL;
+
+	qb_inode = qb_inode_ctx_get (this, fd->inode);
+	if (!qb_inode) {
+		STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->ftruncate, fd, offset,
+			    xdata);
+		return 0;
+	}
+
+	if (qb_local_init (frame) != 0)
+		goto enomem;
+
+	qb_local = frame->local;
+
+	qb_local->inode = inode_ref (fd->inode);
+	qb_local->fd = fd_ref (fd);
+
+	qb_local->stub = fop_ftruncate_stub (frame, NULL, fd, offset, xdata);
+
+	if (!qb_local->stub)
+		goto enomem;
+
+	qb_coroutine (frame, qb_co_truncate);
+
+	return 0;
+enomem:
+	QB_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, 0, 0, 0);
+	return 0;
+}
+
+
+int
+qb_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	     int op_ret, int op_errno, struct iatt *iatt, dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, iatt);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (stat, frame, op_ret, op_errno, iatt, xdata);
+
+	return 0;
+}
+
+int
+qb_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, loc->inode))
+		frame->local = inode_ref (loc->inode);
+
+	STACK_WIND (frame, qb_stat_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->stat, loc, xdata);
+	return 0;
+}
+
+
+int
+qb_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	      int op_ret, int op_errno, struct iatt *iatt, dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, iatt);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (fstat, frame, op_ret, op_errno, iatt, xdata);
+
+	return 0;
+}
+
+
+int
+qb_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, fd->inode))
+		frame->local = inode_ref (fd->inode);
+
+	STACK_WIND (frame, qb_fstat_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+	return 0;
+}
+
+
+int
+qb_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+		dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, pre);
+		qb_iatt_fixup (this, inode, post);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (setattr, frame, op_ret, op_errno, pre, post, xdata);
+
+	return 0;
+}
+
+
+int
+qb_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+	    int valid, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, loc->inode))
+		frame->local = inode_ref (loc->inode);
+
+	STACK_WIND (frame, qb_setattr_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->setattr, loc, buf, valid, xdata);
+	return 0;
+}
+
+
+int
+qb_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+		 dict_t *xdata)
+{
+	inode_t *inode = NULL;
+
+	inode = frame->local;
+	frame->local = NULL;
+
+	if (inode) {
+		qb_iatt_fixup (this, inode, pre);
+		qb_iatt_fixup (this, inode, post);
+		inode_unref (inode);
+	}
+
+	QB_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, pre, post, xdata);
+
+	return 0;
+}
+
+
+int
+qb_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+	     int valid, dict_t *xdata)
+{
+	if (qb_inode_ctx_get (this, fd->inode))
+		frame->local = inode_ref (fd->inode);
+
+	STACK_WIND (frame, qb_setattr_cbk, FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fsetattr, fd, buf, valid, xdata);
+	return 0;
+}
+
+
+int
+qb_forget (xlator_t *this, inode_t *inode)
+{
+	return qb_inode_cleanup (this, inode, 0);
+}
+
+
+int
+qb_release (xlator_t *this, fd_t *fd)
+{
+	call_frame_t *frame = NULL;
+
+	frame = create_frame (this, this->ctx->pool);
+	if (!frame) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Could not allocate frame. "
+			"Leaking QEMU BlockDriverState");
+		return -1;
+	}
+
+	if (qb_local_init (frame) != 0) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Could not allocate local. "
+			"Leaking QEMU BlockDriverState");
+		STACK_DESTROY (frame->root);
+		return -1;
+	}
+
+	if (qb_coroutine (frame, qb_co_close) != 0) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"Could not allocate coroutine. "
+			"Leaking QEMU BlockDriverState");
+		qb_local_free (this, frame->local);
+		frame->local = NULL;
+		STACK_DESTROY (frame->root);
+	}
+
+	return 0;
+}
+
+int
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_qb_mt_end + 1);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init "
+                        "failed");
+        return ret;
+}
+
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+	return 0;
+}
+
+
+int
+init (xlator_t *this)
+{
+        qb_conf_t *conf    = NULL;
+        int32_t    ret     = -1;
+	static int bdrv_inited = 0;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "FATAL: qemu-block (%s) not configured with exactly "
+                        "one child", this->name);
+                goto out;
+        }
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_qb_mt_qb_conf_t);
+        if (!conf)
+                goto out;
+
+        /* configure 'option window-size <size>' */
+        GF_OPTION_INIT ("default-password", conf->default_password, str, out);
+
+	/* qemu coroutines use "co_mutex" for synchronizing among themselves.
+	   However "co_mutex" itself is not threadsafe if the coroutine framework
+	   is multithreaded (which usually is not). However synctasks are
+	   fundamentally multithreaded, so for now create a syncenv which has
+	   scaling limits set to max 1 thread so that the qemu coroutines can
+	   execute "safely".
+
+	   Future work: provide an implementation of "co_mutex" which is
+	   threadsafe and use the global multithreaded ctx->env syncenv.
+	*/
+	conf->env = syncenv_new (0, 1, 1);
+
+        this->private = conf;
+
+        ret = 0;
+
+	snprintf (conf->qb_xattr_key, QB_XATTR_KEY_MAX, QB_XATTR_KEY_FMT,
+		  this->name);
+
+	cur_mon = (void *) 1;
+
+	if (!bdrv_inited) {
+		bdrv_init ();
+		bdrv_inited = 1;
+	}
+
+out:
+        if (ret)
+                GF_FREE (conf);
+
+        return ret;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        qb_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        this->private = NULL;
+
+	if (conf->root_inode)
+		inode_unref(conf->root_inode);
+        GF_FREE (conf);
+
+	return;
+}
+
+
+struct xlator_fops fops = {
+	.lookup      = qb_lookup,
+	.fsetxattr   = qb_fsetxattr,
+	.setxattr    = qb_setxattr,
+	.open        = qb_open,
+        .writev      = qb_writev,
+        .readv       = qb_readv,
+        .fsync       = qb_fsync,
+        .truncate    = qb_truncate,
+        .ftruncate   = qb_ftruncate,
+        .stat        = qb_stat,
+        .fstat       = qb_fstat,
+	.setattr     = qb_setattr,
+	.fsetattr    = qb_fsetattr,
+        .flush       = qb_flush,
+/*
+	.getxattr    = qb_getxattr,
+	.fgetxattr   = qb_fgetxattr
+*/
+	.readdirp    = qb_readdirp,
+};
+
+
+struct xlator_cbks cbks = {
+        .forget   = qb_forget,
+	.release  = qb_release,
+};
+
+
+struct xlator_dumpops dumpops = {
+};
+
+
+struct volume_options options[] = {
+        { .key  = {"default-password"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "",
+          .description = "Default password for the AES encrypted block images."
+        },
+        { .key = {NULL} },
+};
diff --git a/xlators/features/qemu-block/src/qemu-block.h b/xlators/features/qemu-block/src/qemu-block.h
new file mode 100644
index 000000000..c95f2799a
--- /dev/null
+++ b/xlators/features/qemu-block/src/qemu-block.h
@@ -0,0 +1,109 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QEMU_BLOCK_H
+#define __QEMU_BLOCK_H
+
+#include "syncop.h"
+#include "call-stub.h"
+#include "block/block_int.h"
+#include "monitor/monitor.h"
+
+/* QB_XATTR_KEY_FMT is the on-disk xattr stored in the inode which
+   indicates that the file must be "interpreted" by the block format
+   logic. The value of the key is of the pattern:
+
+   "format:virtual_size"
+
+   e.g
+
+   "qcow2:20GB" or "qed:100GB"
+
+   The format and virtual size are colon separated. The format is
+   a case sensitive string which qemu recognizes. virtual_size is
+   specified as a size which glusterfs recognizes as size (i.e.,
+   value accepted by gf_string2bytesize())
+*/
+#define QB_XATTR_KEY_FMT "trusted.glusterfs.%s.format"
+
+#define QB_XATTR_KEY_MAX 64
+
+#define QB_XATTR_VAL_MAX 64
+
+
+typedef struct qb_inode {
+	char fmt[QB_XATTR_VAL_MAX]; /* this is only the format, not "format:size" */
+	size_t size; /* virtual size in bytes */
+	BlockDriverState *bs;
+	int refcnt;
+	uuid_t backing_gfid;
+	char *backing_fname;
+} qb_inode_t;
+
+
+typedef struct qb_conf {
+	Monitor *mon;
+	struct syncenv *env;
+	char qb_xattr_key[QB_XATTR_KEY_MAX];
+	char *default_password;
+	inode_t *root_inode;
+} qb_conf_t;
+
+
+typedef struct qb_local {
+	call_frame_t      *frame; /* backpointer */
+	call_stub_t       *stub;
+	inode_t           *inode;
+	fd_t              *fd;
+	char               fmt[QB_XATTR_VAL_MAX+1];
+	char               name[256];
+	synctask_fn_t      synctask_fn;
+	struct list_head   list;
+} qb_local_t;
+
+void qb_local_free (xlator_t *this, qb_local_t *local);
+int qb_coroutine (call_frame_t *frame, synctask_fn_t fn);
+inode_t *qb_inode_from_filename (const char *filename);
+int qb_inode_to_filename (inode_t *inode, char *filename, int size);
+int qb_format_extract (xlator_t *this, char *format, inode_t *inode);
+
+qb_inode_t *qb_inode_ctx_get (xlator_t *this, inode_t *inode);
+
+#define QB_STACK_UNWIND(typ, frame, args ...) do {	\
+	qb_local_t *__local = frame->local;		\
+	xlator_t *__this = frame->this;			\
+							\
+	frame->local = NULL;				\
+	STACK_UNWIND_STRICT (typ, frame, args);		\
+	if (__local)					\
+		qb_local_free (__this, __local);	\
+	} while (0)
+
+#define QB_STUB_UNWIND(stub, op_ret, op_errno) do {	\
+	qb_local_t *__local = stub->frame->local;	\
+	xlator_t *__this = stub->frame->this;		\
+							\
+	stub->frame->local = NULL;			\
+	call_unwind_error (stub, op_ret, op_errno);	\
+	if (__local)					\
+		qb_local_free (__this, __local);	\
+	} while (0)
+
+#define QB_STUB_RESUME(stub_errno) do {			\
+	qb_local_t *__local = stub->frame->local;	\
+	xlator_t *__this = stub->frame->this;		\
+							\
+	stub->frame->local = NULL;			\
+	call_resume (stub);				\
+	if (__local)					\
+		qb_local_free (__this, __local);	\
+	} while (0)
+
+#endif /* !__QEMU_BLOCK_H */
diff --git a/xlators/features/quiesce/Makefile.am b/xlators/features/quiesce/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/quiesce/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/quiesce/src/Makefile.am b/xlators/features/quiesce/src/Makefile.am
new file mode 100644
index 000000000..15e46629e
--- /dev/null
+++ b/xlators/features/quiesce/src/Makefile.am
@@ -0,0 +1,15 @@
+xlator_LTLIBRARIES = quiesce.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+quiesce_la_LDFLAGS = -module -avoid-version
+
+quiesce_la_SOURCES = quiesce.c
+quiesce_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = quiesce.h quiesce-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/quiesce/src/quiesce-mem-types.h b/xlators/features/quiesce/src/quiesce-mem-types.h
new file mode 100644
index 000000000..6e582f424
--- /dev/null
+++ b/xlators/features/quiesce/src/quiesce-mem-types.h
@@ -0,0 +1,20 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QUIESCE_MEM_TYPES_H__
+#define __QUIESCE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_quiesce_mem_types_ {
+        gf_quiesce_mt_priv_t = gf_common_mt_end + 1,
+        gf_quiesce_mt_end
+};
+#endif
diff --git a/xlators/features/quiesce/src/quiesce.c b/xlators/features/quiesce/src/quiesce.c
new file mode 100644
index 000000000..24c7dc6ed
--- /dev/null
+++ b/xlators/features/quiesce/src/quiesce.c
@@ -0,0 +1,2610 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "quiesce.h"
+#include "defaults.h"
+#include "call-stub.h"
+
+/* TODO: */
+/* Think about 'writev/_*_lk/setattr/xattrop/' fops to do re-transmittion */
+
+
+/* Quiesce Specific Functions */
+void
+gf_quiesce_local_wipe (xlator_t *this, quiesce_local_t *local)
+{
+        if (!local || !this || !this->private)
+                return;
+
+        if (local->loc.inode)
+                loc_wipe (&local->loc);
+        if (local->fd)
+                fd_unref (local->fd);
+        GF_FREE (local->name);
+        GF_FREE (local->volname);
+        if (local->dict)
+                dict_unref (local->dict);
+        if (local->iobref)
+                iobref_unref (local->iobref);
+        GF_FREE (local->vector);
+
+        mem_put (local);
+}
+
+call_stub_t *
+gf_quiesce_dequeue (xlator_t *this)
+{
+        call_stub_t  *stub = NULL;
+        quiesce_priv_t *priv = NULL;
+
+        priv = this->private;
+
+        if (!priv || list_empty (&priv->req))
+                return NULL;
+
+        LOCK (&priv->lock);
+        {
+                stub = list_entry (priv->req.next, call_stub_t, list);
+                list_del_init (&stub->list);
+                priv->queue_size--;
+        }
+        UNLOCK (&priv->lock);
+
+        return stub;
+}
+
+void *
+gf_quiesce_dequeue_start (void *data)
+{
+        xlator_t       *this = NULL;
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        this = data;
+        priv = this->private;
+        THIS = this;
+
+        while (!list_empty (&priv->req)) {
+                stub = gf_quiesce_dequeue (this);
+                if (stub) {
+                        call_resume (stub);
+                }
+        }
+
+        return 0;
+}
+
+
+void
+gf_quiesce_timeout (void *data)
+{
+        xlator_t       *this = NULL;
+        quiesce_priv_t *priv = NULL;
+
+        this = data;
+        priv = this->private;
+        THIS = this;
+
+        LOCK (&priv->lock);
+        {
+                priv->pass_through = _gf_true;
+        }
+        UNLOCK (&priv->lock);
+
+        gf_quiesce_dequeue_start (this);
+
+        return;
+}
+
+void
+gf_quiesce_enqueue (xlator_t *this, call_stub_t *stub)
+{
+        quiesce_priv_t *priv    = NULL;
+        struct timespec timeout = {0,};
+
+        priv = this->private;
+        if (!priv) {
+                gf_log_callingfn (this->name, GF_LOG_ERROR,
+                                  "this->private == NULL");
+                return;
+        }
+
+        LOCK (&priv->lock);
+        {
+                list_add_tail (&stub->list, &priv->req);
+                priv->queue_size++;
+        }
+        UNLOCK (&priv->lock);
+
+        if (!priv->timer) {
+                timeout.tv_sec = 20;
+                timeout.tv_nsec = 0;
+
+                priv->timer = gf_timer_call_after (this->ctx,
+                                                   timeout,
+                                                   gf_quiesce_timeout,
+                                                   (void *) this);
+        }
+
+        return;
+}
+
+
+
+/* _CBK function section */
+
+int32_t
+quiesce_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *dict, struct iatt *postparent)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_lookup_stub (frame, default_lookup_resume,
+                                        &local->loc, local->dict);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM,
+                                             NULL, NULL, NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             dict, postparent);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_stat_stub (frame, default_stat_resume,
+                                      &local->loc, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_access_stub (frame, default_access_resume,
+                                        &local->loc, local->flag, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, const char *path,
+                      struct iatt *buf, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_readlink_stub (frame, default_readlink_resume,
+                                          &local->loc, local->size, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM,
+                                             NULL, NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_open_stub (frame, default_open_resume,
+                                      &local->loc, local->flag, local->fd,
+                                      xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (open, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_readv_stub (frame, default_readv_resume,
+                                       local->fd, local->size, local->offset,
+                                       local->io_flag, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
+                                             NULL, 0, NULL, NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
+                             stbuf, iobref, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_flush_stub (frame, default_flush_resume,
+                                       local->fd, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+
+
+int32_t
+quiesce_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_fsync_stub (frame, default_fsync_resume,
+                                       local->fd, local->flag, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM,
+                                             NULL, NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_fstat_stub (frame, default_fstat_resume,
+                                       local->fd, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_opendir_stub (frame, default_opendir_resume,
+                                         &local->loc, local->fd, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume,
+                                          local->fd, local->flag, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_statfs_stub (frame, default_statfs_resume,
+                                        &local->loc, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume,
+                                           local->fd, local->name, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_getxattr_stub (frame, default_getxattr_resume,
+                                          &local->loc, local->name, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                       uint8_t *strong_checksum, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
+                                           local->fd, local->offset, local->flag, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM,
+                                             0, NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
+                             strong_checksum, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_readdir_stub (frame, default_readdir_resume,
+                                         local->fd, local->size, local->offset, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_readdirp_stub (frame, default_readdirp_resume,
+                                          local->fd, local->size, local->offset,
+                                          local->dict);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+
+#if 0
+
+int32_t
+quiesce_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_writev_stub (frame, default_writev_resume,
+                                        local->fd, local->vector, local->flag,
+                                        local->offset, local->io_flags,
+                                        local->iobref, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM,
+                                             NULL, NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_xattrop_stub (frame, default_xattrop_resume,
+                                         &local->loc, local->xattrop_flags,
+                                         local->dict, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_fxattrop_stub (frame, default_fxattrop_resume,
+                                          local->fd, local->xattrop_flags,
+                                          local->dict, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_lk_stub (frame, default_lk_resume,
+                                    local->fd, local->flag, &local->flock, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM,
+                                             NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_inodelk_stub (frame, default_inodelk_resume,
+                                         local->volname, &local->loc,
+                                         local->flag, &local->flock, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_finodelk_stub (frame, default_finodelk_resume,
+                                         local->volname, local->fd,
+                                         local->flag, &local->flock, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_entrylk_stub (frame, default_entrylk_resume,
+                                         local->volname, &local->loc,
+                                         local->name, local->cmd, local->type, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_fentrylk_stub (frame, default_fentrylk_resume,
+                                          local->volname, local->fd,
+                                          local->name, local->cmd, local->type, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_setattr_stub (frame, default_setattr_resume,
+                                         &local->loc, &local->stbuf, local->flag, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM,
+                                             NULL, NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
+                             statpost, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+int32_t
+quiesce_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                      struct iatt *statpost, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        local = frame->local;
+        frame->local = NULL;
+
+        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+                /* Re-transmit (by putting in the queue) */
+                stub = fop_fsetattr_stub (frame, default_fsetattr_resume,
+                                          local->fd, &local->stbuf, local->flag, xdata);
+                if (!stub) {
+                        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM,
+                                             NULL, NULL, NULL);
+                        goto out;
+                }
+
+                gf_quiesce_enqueue (this, stub);
+                goto out;
+        }
+
+        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
+                             statpost, xdata);
+out:
+        gf_quiesce_local_wipe (this, local);
+
+        return 0;
+}
+
+#endif /* if 0 */
+
+
+/* FOP */
+
+/* No retransmittion */
+
+int32_t
+quiesce_removexattr (call_frame_t *frame,
+		     xlator_t *this,
+		     loc_t *loc,
+		     const char *name, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame,
+                            default_removexattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->removexattr,
+                            loc,
+                            name, xdata);
+                return 0;
+        }
+
+        stub = fop_removexattr_stub (frame, default_removexattr_resume,
+                                     loc, name, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_truncate (call_frame_t *frame,
+		  xlator_t *this,
+		  loc_t *loc,
+		  off_t offset, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame,
+                            default_truncate_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->truncate,
+                            loc,
+                            offset, xdata);
+                return 0;
+        }
+
+        stub = fop_truncate_stub (frame, default_truncate_resume, loc, offset, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_fsetxattr (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t *fd,
+                   dict_t *dict,
+                   int32_t flags, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame,
+                            default_fsetxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsetxattr,
+                            fd,
+                            dict,
+                            flags, xdata);
+                return 0;
+        }
+
+        stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume,
+                                   fd, dict, flags, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_setxattr (call_frame_t *frame,
+		  xlator_t *this,
+		  loc_t *loc,
+		  dict_t *dict,
+		  int32_t flags, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame,
+                            default_setxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            loc,
+                            dict,
+                            flags, xdata);
+                return 0;
+        }
+
+        stub = fop_setxattr_stub (frame, default_setxattr_resume,
+                                  loc, dict, flags, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_create (call_frame_t *frame, xlator_t *this,
+		loc_t *loc, int32_t flags, mode_t mode,
+                mode_t umask, fd_t *fd, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                /* Don't send O_APPEND below, as write() re-transmittions can
+                   fail with O_APPEND */
+                STACK_WIND (frame, default_create_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->create,
+                            loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
+                return 0;
+        }
+
+        stub = fop_create_stub (frame, default_create_resume,
+                                loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (create, frame, -1, ENOMEM,
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_link (call_frame_t *frame,
+	      xlator_t *this,
+	      loc_t *oldloc,
+	      loc_t *newloc, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame,
+                            default_link_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->link,
+                            oldloc, newloc, xdata);
+                return 0;
+        }
+
+        stub = fop_link_stub (frame, default_link_resume, oldloc, newloc, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (link, frame, -1, ENOMEM,
+                                     NULL, NULL, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_rename (call_frame_t *frame,
+		xlator_t *this,
+		loc_t *oldloc,
+		loc_t *newloc, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame,
+                            default_rename_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->rename,
+                            oldloc, newloc, xdata);
+                return 0;
+        }
+
+        stub = fop_rename_stub (frame, default_rename_resume, oldloc, newloc, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM,
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int
+quiesce_symlink (call_frame_t *frame, xlator_t *this,
+		 const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame, default_symlink_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->symlink,
+                            linkpath, loc, umask, xdata);
+                return 0;
+        }
+
+        stub = fop_symlink_stub (frame, default_symlink_resume,
+                                 linkpath, loc, umask, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (symlink, frame, -1, ENOMEM,
+                                     NULL, NULL, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int
+quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame, default_rmdir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->rmdir,
+                            loc, flags, xdata);
+                return 0;
+        }
+
+        stub = fop_rmdir_stub (frame, default_rmdir_resume, loc, flags, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_unlink (call_frame_t *frame,
+		xlator_t *this,
+		loc_t *loc, int xflag, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame,
+                            default_unlink_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->unlink,
+                            loc, xflag, xdata);
+                return 0;
+        }
+
+        stub = fop_unlink_stub (frame, default_unlink_resume, loc, xflag, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int
+quiesce_mkdir (call_frame_t *frame, xlator_t *this,
+	       loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame, default_mkdir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->mkdir,
+                            loc, mode, umask, xdata);
+                return 0;
+        }
+
+        stub = fop_mkdir_stub (frame, default_mkdir_resume,
+                               loc, mode, umask, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (mkdir, frame, -1, ENOMEM,
+                                     NULL, NULL, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int
+quiesce_mknod (call_frame_t *frame, xlator_t *this,
+	       loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame, default_mknod_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->mknod,
+                            loc, mode, rdev, umask, xdata);
+                return 0;
+        }
+
+        stub = fop_mknod_stub (frame, default_mknod_resume,
+                               loc, mode, rdev, umask, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (mknod, frame, -1, ENOMEM,
+                                     NULL, NULL, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_ftruncate (call_frame_t *frame,
+		   xlator_t *this,
+		   fd_t *fd,
+		   off_t offset, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv->pass_through) {
+                STACK_WIND (frame,
+                            default_ftruncate_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->ftruncate,
+                            fd,
+                            offset, xdata);
+                return 0;
+        }
+
+        stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+/* Re-transmittion */
+
+int32_t
+quiesce_readlink (call_frame_t *frame,
+		  xlator_t *this,
+		  loc_t *loc,
+		  size_t size, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                loc_dup (loc, &local->loc);
+                local->size = size;
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_readlink_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->readlink,
+                            loc,
+                            size, xdata);
+                return 0;
+        }
+
+        stub = fop_readlink_stub (frame, default_readlink_resume, loc, size, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_access (call_frame_t *frame,
+		xlator_t *this,
+		loc_t *loc,
+		int32_t mask, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                loc_dup (loc, &local->loc);
+                local->flag = mask;
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_access_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->access,
+                            loc,
+                            mask, xdata);
+                return 0;
+        }
+
+        stub = fop_access_stub (frame, default_access_resume, loc, mask, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_fgetxattr (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                local->fd = fd_ref (fd);
+                if (name)
+                        local->name = gf_strdup (name);
+
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_fgetxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fgetxattr,
+                            fd,
+                            name, xdata);
+                return 0;
+        }
+
+        stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_statfs (call_frame_t *frame,
+		xlator_t *this,
+		loc_t *loc, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                loc_dup (loc, &local->loc);
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_statfs_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->statfs,
+                            loc, xdata);
+                return 0;
+        }
+
+        stub = fop_statfs_stub (frame, default_statfs_resume, loc, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_fsyncdir (call_frame_t *frame,
+		  xlator_t *this,
+		  fd_t *fd,
+		  int32_t flags, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                local->fd = fd_ref (fd);
+                local->flag = flags;
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_fsyncdir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsyncdir,
+                            fd,
+                            flags, xdata);
+                return 0;
+        }
+
+        stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume, fd, flags, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_opendir (call_frame_t *frame,
+		 xlator_t *this,
+		 loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                loc_dup (loc, &local->loc);
+                local->fd = fd_ref (fd);
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_opendir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->opendir,
+                            loc, fd, xdata);
+                return 0;
+        }
+
+        stub = fop_opendir_stub (frame, default_opendir_resume, loc, fd, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_fstat (call_frame_t *frame,
+	       xlator_t *this,
+	       fd_t *fd, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                local->fd = fd_ref (fd);
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_fstat_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fstat,
+                            fd, xdata);
+                return 0;
+        }
+
+        stub = fop_fstat_stub (frame, default_fstat_resume, fd, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_fsync (call_frame_t *frame,
+	       xlator_t *this,
+	       fd_t *fd,
+	       int32_t flags, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                local->fd = fd_ref (fd);
+                local->flag = flags;
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_fsync_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsync,
+                            fd,
+                            flags, xdata);
+                return 0;
+        }
+
+        stub = fop_fsync_stub (frame, default_fsync_resume, fd, flags, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_flush (call_frame_t *frame,
+	       xlator_t *this,
+	       fd_t *fd, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                local->fd = fd_ref (fd);
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_flush_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->flush,
+                            fd, xdata);
+                return 0;
+        }
+
+        stub = fop_flush_stub (frame, default_flush_resume, fd, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_writev (call_frame_t *frame,
+		xlator_t *this,
+		fd_t *fd,
+		struct iovec *vector,
+		int32_t count,
+		off_t off, uint32_t flags,
+                struct iobref *iobref, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame,
+                            default_writev_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->writev,
+                            fd,
+                            vector,
+                            count,
+                            off, flags,
+                            iobref, xdata);
+                return 0;
+        }
+
+        stub = fop_writev_stub (frame, default_writev_resume,
+                                fd, vector, count, off, flags, iobref, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_readv (call_frame_t *frame,
+	       xlator_t *this,
+	       fd_t *fd,
+	       size_t size,
+	       off_t offset, uint32_t flags, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                local->fd = fd_ref (fd);
+                local->size = size;
+                local->offset = offset;
+                local->io_flag = flags;
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_readv_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->readv,
+                            fd,
+                            size,
+                            offset, flags, xdata);
+                return 0;
+        }
+
+        stub = fop_readv_stub (frame, default_readv_resume, fd, size, offset,
+                               flags, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
+                                     NULL, 0, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_open (call_frame_t *frame,
+	      xlator_t *this,
+	      loc_t *loc,
+	      int32_t flags, fd_t *fd,
+              dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                loc_dup (loc, &local->loc);
+                local->fd = fd_ref (fd);
+
+                /* Don't send O_APPEND below, as write() re-transmittions can
+                   fail with O_APPEND */
+                local->flag = (flags & ~O_APPEND);
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_open_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->open,
+                            loc, (flags & ~O_APPEND), fd, xdata);
+                return 0;
+        }
+
+        stub = fop_open_stub (frame, default_open_resume, loc,
+                              (flags & ~O_APPEND), fd, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_getxattr (call_frame_t *frame,
+		  xlator_t *this,
+		  loc_t *loc,
+		  const char *name, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                loc_dup (loc, &local->loc);
+                if (name)
+                        local->name = gf_strdup (name);
+
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_getxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->getxattr,
+                            loc,
+                            name, xdata);
+                return 0;
+        }
+
+        stub = fop_getxattr_stub (frame, default_getxattr_resume, loc, name, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_xattrop (call_frame_t *frame,
+		 xlator_t *this,
+		 loc_t *loc,
+		 gf_xattrop_flags_t flags,
+		 dict_t *dict, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame,
+                            default_xattrop_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->xattrop,
+                            loc,
+                            flags,
+                            dict, xdata);
+                return 0;
+        }
+
+        stub = fop_xattrop_stub (frame, default_xattrop_resume,
+                                 loc, flags, dict, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_fxattrop (call_frame_t *frame,
+		  xlator_t *this,
+		  fd_t *fd,
+		  gf_xattrop_flags_t flags,
+		  dict_t *dict, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame,
+                            default_fxattrop_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fxattrop,
+                            fd,
+                            flags,
+                            dict, xdata);
+                return 0;
+        }
+
+        stub = fop_fxattrop_stub (frame, default_fxattrop_resume,
+                                  fd, flags, dict, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_lk (call_frame_t *frame,
+	    xlator_t *this,
+	    fd_t *fd,
+	    int32_t cmd,
+	    struct gf_flock *lock, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame,
+                            default_lk_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->lk,
+                            fd,
+                            cmd,
+                            lock, xdata);
+                return 0;
+        }
+
+        stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, lock, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_inodelk (call_frame_t *frame, xlator_t *this,
+		 const char *volume, loc_t *loc, int32_t cmd,
+                 struct gf_flock *lock, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame,
+                            default_inodelk_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->inodelk,
+                            volume, loc, cmd, lock, xdata);
+                return 0;
+        }
+
+        stub = fop_inodelk_stub (frame, default_inodelk_resume,
+                                 volume, loc, cmd, lock, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_finodelk (call_frame_t *frame, xlator_t *this,
+		  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame,
+                            default_finodelk_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->finodelk,
+                            volume, fd, cmd, lock, xdata);
+                return 0;
+        }
+
+        stub = fop_finodelk_stub (frame, default_finodelk_resume,
+                                  volume, fd, cmd, lock, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_entrylk (call_frame_t *frame, xlator_t *this,
+		 const char *volume, loc_t *loc, const char *basename,
+		 entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame, default_entrylk_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->entrylk,
+                            volume, loc, basename, cmd, type, xdata);
+                return 0;
+        }
+
+        stub = fop_entrylk_stub (frame, default_entrylk_resume,
+                                 volume, loc, basename, cmd, type, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_fentrylk (call_frame_t *frame, xlator_t *this,
+		  const char *volume, fd_t *fd, const char *basename,
+		  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame, default_fentrylk_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fentrylk,
+                            volume, fd, basename, cmd, type, xdata);
+                return 0;
+        }
+
+        stub = fop_fentrylk_stub (frame, default_fentrylk_resume,
+                                  volume, fd, basename, cmd, type, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_rchecksum (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t *fd, off_t offset,
+                   int32_t len, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                local->fd = fd_ref (fd);
+                local->offset = offset;
+                local->flag = len;
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_rchecksum_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->rchecksum,
+                            fd, offset, len, xdata);
+                return 0;
+        }
+
+        stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
+                                   fd, offset, len, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM, 0, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_readdir (call_frame_t *frame,
+		 xlator_t *this,
+		 fd_t *fd,
+		 size_t size,
+		 off_t off, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                local->fd = fd_ref (fd);
+                local->size = size;
+                local->offset = off;
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_readdir_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->readdir,
+                            fd, size, off, xdata);
+                return 0;
+        }
+
+        stub = fop_readdir_stub (frame, default_readdir_resume, fd, size, off, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_readdirp (call_frame_t *frame,
+		  xlator_t *this,
+		  fd_t *fd,
+		  size_t size,
+		  off_t off, dict_t *dict)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                local->fd = fd_ref (fd);
+                local->size = size;
+                local->offset = off;
+                local->dict = dict_ref (dict);
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_readdirp_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->readdirp,
+                            fd, size, off, dict);
+                return 0;
+        }
+
+        stub = fop_readdirp_stub (frame, default_readdirp_resume, fd, size,
+                                  off, dict);
+        if (!stub) {
+                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_setattr (call_frame_t *frame,
+                 xlator_t *this,
+                 loc_t *loc,
+                 struct iatt *stbuf,
+                 int32_t valid, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame,
+                            default_setattr_cbk,
+                            FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->setattr,
+                            loc, stbuf, valid, xdata);
+                return 0;
+        }
+
+        stub = fop_setattr_stub (frame, default_setattr_resume,
+                                   loc, stbuf, valid, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+
+int32_t
+quiesce_stat (call_frame_t *frame,
+	      xlator_t *this,
+	      loc_t *loc, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                loc_dup (loc, &local->loc);
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_stat_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->stat,
+                            loc, xdata);
+                return 0;
+        }
+
+        stub = fop_stat_stub (frame, default_stat_resume, loc, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_lookup (call_frame_t *frame,
+		xlator_t *this,
+		loc_t *loc,
+		dict_t *xattr_req)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+        quiesce_local_t *local = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                local = mem_get0 (priv->local_pool);
+                loc_dup (loc, &local->loc);
+                local->dict = dict_ref (xattr_req);
+                frame->local = local;
+
+                STACK_WIND (frame,
+                            quiesce_lookup_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->lookup,
+                            loc, xattr_req);
+                return 0;
+        }
+
+        stub = fop_lookup_stub (frame, default_lookup_resume, loc, xattr_req);
+        if (!stub) {
+                STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM,
+                                     NULL, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+quiesce_fsetattr (call_frame_t *frame,
+                  xlator_t *this,
+                  fd_t *fd,
+                  struct iatt *stbuf,
+                  int32_t valid, dict_t *xdata)
+{
+	quiesce_priv_t *priv = NULL;
+        call_stub_t    *stub = NULL;
+
+        priv = this->private;
+
+        if (priv && priv->pass_through) {
+                STACK_WIND (frame,
+                            default_fsetattr_cbk,
+                            FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->fsetattr,
+                            fd, stbuf, valid, xdata);
+                return 0;
+        }
+
+        stub = fop_fsetattr_stub (frame, default_fsetattr_resume,
+                                  fd, stbuf, valid, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+                return 0;
+        }
+
+        gf_quiesce_enqueue (this, stub);
+
+        return 0;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_quiesce_mt_end + 1);
+
+        return ret;
+}
+
+int
+init (xlator_t *this)
+{
+        int ret = -1;
+        quiesce_priv_t *priv = NULL;
+
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"'quiesce' not configured with exactly one child");
+                goto out;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_quiesce_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        priv->local_pool =  mem_pool_new (quiesce_local_t,
+                                          GF_FOPS_EXPECTED_IN_PARALLEL);
+
+        LOCK_INIT (&priv->lock);
+        priv->pass_through = _gf_false;
+
+        INIT_LIST_HEAD (&priv->req);
+
+        this->private = priv;
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        quiesce_priv_t *priv = NULL;
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+        this->private = NULL;
+
+        mem_pool_destroy (priv->local_pool);
+        LOCK_DESTROY (&priv->lock);
+        GF_FREE (priv);
+out:
+        return;
+}
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+        int             ret     = 0;
+        quiesce_priv_t *priv    = NULL;
+        struct timespec timeout = {0,};
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+
+        switch (event) {
+        case GF_EVENT_CHILD_UP:
+        {
+                ret = pthread_create (&priv->thr, NULL, gf_quiesce_dequeue_start,
+                                      this);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to create the quiesce-dequeue thread");
+                }
+
+                LOCK (&priv->lock);
+                {
+                        priv->pass_through = _gf_true;
+                }
+                UNLOCK (&priv->lock);
+                break;
+        }
+        case GF_EVENT_CHILD_DOWN:
+                LOCK (&priv->lock);
+                {
+                        priv->pass_through = _gf_false;
+                }
+                UNLOCK (&priv->lock);
+
+                if (priv->timer)
+                        break;
+                timeout.tv_sec = 20;
+                timeout.tv_nsec = 0;
+
+                priv->timer = gf_timer_call_after (this->ctx,
+                                                   timeout,
+                                                   gf_quiesce_timeout,
+                                                   (void *) this);
+
+                if (priv->timer == NULL) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Cannot create timer");
+                }
+
+                break;
+        default:
+                break;
+        }
+
+        ret = default_notify (this, event, data);
+out:
+        return ret;
+}
+
+
+struct xlator_fops fops = {
+        /* write/modifying fops */
+	.mknod       = quiesce_mknod,
+	.create      = quiesce_create,
+	.truncate    = quiesce_truncate,
+	.ftruncate   = quiesce_ftruncate,
+	.setxattr    = quiesce_setxattr,
+	.removexattr = quiesce_removexattr,
+	.symlink     = quiesce_symlink,
+	.unlink      = quiesce_unlink,
+	.link        = quiesce_link,
+	.mkdir       = quiesce_mkdir,
+	.rmdir       = quiesce_rmdir,
+	.rename      = quiesce_rename,
+
+        /* The below calls are known to change state, hence
+           re-transmittion is not advised */
+	.lk          = quiesce_lk,
+	.inodelk     = quiesce_inodelk,
+	.finodelk    = quiesce_finodelk,
+	.entrylk     = quiesce_entrylk,
+	.fentrylk    = quiesce_fentrylk,
+	.xattrop     = quiesce_xattrop,
+	.fxattrop    = quiesce_fxattrop,
+        .setattr     = quiesce_setattr,
+        .fsetattr    = quiesce_fsetattr,
+
+        /* Special case, re-transmittion is not harmful *
+         * as offset is properly sent from above layers */
+        /* TODO: not re-transmitted as of now */
+	.writev      = quiesce_writev,
+
+        /* re-transmittable fops */
+	.lookup      = quiesce_lookup,
+	.stat        = quiesce_stat,
+	.fstat       = quiesce_fstat,
+	.access      = quiesce_access,
+	.readlink    = quiesce_readlink,
+	.getxattr    = quiesce_getxattr,
+	.open        = quiesce_open,
+	.readv       = quiesce_readv,
+	.flush       = quiesce_flush,
+	.fsync       = quiesce_fsync,
+	.statfs      = quiesce_statfs,
+	.opendir     = quiesce_opendir,
+	.readdir     = quiesce_readdir,
+	.readdirp    = quiesce_readdirp,
+	.fsyncdir    = quiesce_fsyncdir,
+
+};
+
+struct xlator_dumpops dumpops;
+
+
+struct xlator_cbks cbks;
+
+
+struct volume_options options[] = {
+	{ .key  = {NULL} },
+};
diff --git a/xlators/features/quiesce/src/quiesce.h b/xlators/features/quiesce/src/quiesce.h
new file mode 100644
index 000000000..878ed77e9
--- /dev/null
+++ b/xlators/features/quiesce/src/quiesce.h
@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QUIESCE_H__
+#define __QUIESCE_H__
+
+#include "quiesce-mem-types.h"
+#include "xlator.h"
+#include "timer.h"
+
+#define GF_FOPS_EXPECTED_IN_PARALLEL 512
+
+typedef struct {
+        gf_timer_t       *timer;
+        gf_boolean_t      pass_through;
+        gf_lock_t         lock;
+        struct list_head  req;
+        int               queue_size;
+        pthread_t         thr;
+        struct mem_pool  *local_pool;
+} quiesce_priv_t;
+
+typedef struct {
+        fd_t               *fd;
+        char               *name;
+        char               *volname;
+        loc_t               loc;
+        off_t               size;
+        off_t               offset;
+        mode_t              mode;
+        int32_t             flag;
+        struct iatt         stbuf;
+        struct iovec       *vector;
+        struct iobref      *iobref;
+        dict_t             *dict;
+        struct gf_flock     flock;
+        entrylk_cmd         cmd;
+        entrylk_type        type;
+        gf_xattrop_flags_t  xattrop_flags;
+        int32_t             wbflags;
+        uint32_t            io_flag;
+} quiesce_local_t;
+
+#endif
diff --git a/xlators/features/quota/src/Makefile.am b/xlators/features/quota/src/Makefile.am
index 886d83964..9546f4276 100644
--- a/xlators/features/quota/src/Makefile.am
+++ b/xlators/features/quota/src/Makefile.am
@@ -1,13 +1,17 @@
 xlator_LTLIBRARIES = quota.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-quota_la_LDFLAGS = -module -avoidversion
+quota_la_LDFLAGS = -module -avoid-version
 
 quota_la_SOURCES = quota.c
-quota_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
+quota_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = quota-mem-types.h quota.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/cluster/dht/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/quota/src/quota-mem-types.h b/xlators/features/quota/src/quota-mem-types.h
new file mode 100644
index 000000000..3082865da
--- /dev/null
+++ b/xlators/features/quota/src/quota-mem-types.h
@@ -0,0 +1,27 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __QUOTA_MEM_TYPES_H__
+#define __QUOTA_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_quota_mem_types_ {
+        gf_quota_mt_quota_priv_t = gf_common_mt_end + 1,
+        gf_quota_mt_quota_inode_ctx_t,
+        gf_quota_mt_loc_t,
+        gf_quota_mt_char,
+        gf_quota_mt_int64_t,
+        gf_quota_mt_int32_t,
+        gf_quota_mt_limits_t,
+        gf_quota_mt_quota_dentry_t,
+        gf_quota_mt_end
+};
+#endif
+
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index c898899b5..c527e7ca7 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -1,1056 +1,3503 @@
 /*
-  Copyright (c) 2008, 2009 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
+#include <fnmatch.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include "quota.h"
+#include "common-utils.h"
+#include "defaults.h"
 
-#include <sys/time.h>
+int32_t
+quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
+                   char *name, uuid_t par);
+struct volume_options options[];
 
-#include "xlator.h"
-#include "defaults.h"
-#include "common-utils.h"
+int
+quota_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+{
+        int ret = -1;
 
-struct quota_local {
-	struct stat    stbuf;
-	inode_t       *inode;
-	char          *path;
-	fd_t          *fd;
-	off_t          offset;
-	int32_t        count;
-	struct iovec  *vector;
-	dict_t        *refs;
-	loc_t          loc;
-};
+        if (!loc) {
+                return ret;
+        }
 
+        if (inode) {
+                loc->inode = inode_ref (inode);
+        }
 
-struct quota_priv {
-	char       only_first_time;          /* Used to make sure a call is done only one time */
-	gf_lock_t  lock;                     /* Used while updating variables */
+        if (parent) {
+                loc->parent = inode_ref (parent);
+        }
 
-	uint64_t   disk_usage_limit;         /* Used for Disk usage quota */
-	uint64_t   current_disk_usage;       /* Keep the current usage value */
+        loc->path = gf_strdup (path);
+        if (!loc->path) {
+                goto loc_wipe;
+        }
 
-	uint32_t   min_free_disk_limit;        /* user specified limit, in %*/
-	uint32_t   current_free_disk;          /* current free disk space available, in % */
-	uint32_t   refresh_interval;           /* interval in seconds */
-	uint32_t   min_disk_last_updated_time; /* used for interval calculation */	
-};
+        loc->name = strrchr (loc->path, '/');
+        if (loc->name) {
+                loc->name++;
+        } else {
+                goto loc_wipe;
+        }
+
+        ret = 0;
+
+loc_wipe:
+        if (ret < 0) {
+                loc_wipe (loc);
+        }
+
+        return ret;
+}
 
 
 int
-quota_statvfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf)
-{
-	struct quota_priv *priv = this->private;
-	
-	if (op_ret >= 0) {
-		priv->current_free_disk =
-			(stbuf->f_bavail * 100) / stbuf->f_blocks;
-	}
+quota_inode_loc_fill (inode_t *inode, loc_t *loc)
+{
+        char            *resolvedpath = NULL;
+        inode_t         *parent       = NULL;
+        int              ret          = -1;
+        xlator_t        *this         = NULL;
+
+        if ((!inode) || (!loc)) {
+                return ret;
+        }
+
+        this = THIS;
+
+        if ((inode) && __is_root_gfid (inode->gfid)) {
+                loc->parent = NULL;
+                goto ignore_parent;
+        }
+
+        parent = inode_parent (inode, 0, NULL);
+        if (!parent) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "cannot find parent for inode (gfid:%s)",
+                        uuid_utoa (inode->gfid));
+                goto err;
+        }
+
+ignore_parent:
+        ret = inode_path (inode, NULL, &resolvedpath);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "cannot construct path for inode (gfid:%s)",
+                        uuid_utoa (inode->gfid));
+                goto err;
+        }
+
+        ret = quota_loc_fill (loc, inode, parent, resolvedpath);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "cannot fill loc");
+                goto err;
+        }
+
+err:
+        if (parent) {
+                inode_unref (parent);
+        }
+
+        GF_FREE (resolvedpath);
 
-	STACK_DESTROY (frame->root);
-	return 0;
+        return ret;
 }
 
 
-static void
-build_root_loc (xlator_t *this, loc_t *loc)
+int32_t
+quota_local_cleanup (xlator_t *this, quota_local_t *local)
 {
-	loc->path = "/";
+        if (local == NULL) {
+                goto out;
+        }
+
+        loc_wipe (&local->loc);
+        loc_wipe (&local->newloc);
+        loc_wipe (&local->oldloc);
+        loc_wipe (&local->validate_loc);
+
+        inode_unref (local->inode);
+        LOCK_DESTROY (&local->lock);
+
+        mem_put (local);
+out:
+        return 0;
 }
 
 
-void
-gf_quota_usage_subtract (xlator_t *this, size_t size)
+static inline quota_local_t *
+quota_local_new ()
 {
-	struct quota_priv *priv = NULL;
+        quota_local_t *local = NULL;
+        local = mem_get0 (THIS->local_pool);
+        if (local)
+                LOCK_INIT (&local->lock);
+        return local;
+}
 
-	priv = this->private;
 
-	LOCK (&priv->lock);
-	{
-		if (priv->current_disk_usage < size)
-			priv->current_disk_usage = 0;
-		else
-			priv->current_disk_usage -= size;
-	}
-	UNLOCK (&priv->lock);
+quota_dentry_t *
+__quota_dentry_new (quota_inode_ctx_t *ctx, char *name, uuid_t par)
+{
+        quota_dentry_t    *dentry = NULL;
+        GF_UNUSED int32_t  ret    = 0;
+
+        QUOTA_ALLOC_OR_GOTO (dentry, quota_dentry_t, err);
+
+        INIT_LIST_HEAD (&dentry->next);
+
+        dentry->name = gf_strdup (name);
+        if (dentry->name == NULL) {
+                GF_FREE (dentry);
+                goto err;
+        }
+
+        uuid_copy (dentry->par, par);
+
+        list_add_tail (&dentry->next, &ctx->parents);
+err:
+        return dentry;
 }
 
 
 void
-gf_quota_usage_add (xlator_t *this, size_t size)
+__quota_dentry_free (quota_dentry_t *dentry)
 {
-	struct quota_priv *priv = this->private;
+        if (dentry == NULL) {
+                goto out;
+        }
 
-	LOCK (&priv->lock);
-	{
-		priv->current_disk_usage += size;
-	}
-	UNLOCK (&priv->lock);
+        list_del_init (&dentry->next);
+
+        GF_FREE (dentry->name);
+        GF_FREE (dentry);
+out:
+        return;
 }
 
 
-void 
-gf_quota_update_current_free_disk (xlator_t *this)
+int32_t
+quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
-	call_frame_t *frame = NULL;
-	call_pool_t   *pool = NULL;
-	loc_t          loc;
+        quota_local_t     *local          = NULL;
+        uint32_t           validate_count = 0, link_count = 0;
+        int32_t            ret            = 0;
+        quota_inode_ctx_t *ctx            = NULL;
+        int64_t           *size           = 0;
+        uint64_t           value          = 0;
+        call_stub_t       *stub           = NULL;
+
+        local = frame->local;
+
+        if (op_ret < 0) {
+                goto unwind;
+        }
+
+        GF_ASSERT (local);
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, unwind, op_errno,
+                                        EINVAL);
+        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, dict, unwind, op_errno,
+                                        EINVAL);
+
+        ret = inode_ctx_get (local->validate_loc.inode, this, &value);
+
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+        if ((ret == -1) || (ctx == NULL)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context is not present in inode (gfid:%s)",
+                        uuid_utoa (local->validate_loc.inode->gfid));
+                op_errno = EINVAL;
+                goto unwind;
+        }
+
+        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "size key not present in dict");
+                op_errno = EINVAL;
+                goto unwind;
+        }
 
-	pool  = this->ctx->pool;
-	frame = create_frame (this, pool);
-  
-	build_root_loc (this, &loc);
+        local->just_validated = 1; /* so that we don't go into infinite
+                                    * loop of validation and checking
+                                    * limit when timeout is zero.
+                                    */
+        LOCK (&ctx->lock);
+        {
+                ctx->size = ntoh64 (*size);
+                gettimeofday (&ctx->tv, NULL);
+        }
+        UNLOCK (&ctx->lock);
+
+        quota_check_limit (frame, local->validate_loc.inode, this, NULL, NULL);
+        return 0;
+
+unwind:
+        LOCK (&local->lock);
+        {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
 
-	STACK_WIND (frame, quota_statvfs_cbk,
-		    this->children->xlator,
-		    this->children->xlator->fops->statfs, &loc);
+                validate_count = --local->validate_count;
+                link_count = local->link_count;
 
-	return ;
+                if ((validate_count == 0) && (link_count == 0)) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+
+        return 0;
 }
 
 
-int
-gf_quota_check_free_disk (xlator_t *this) 
-{
-        struct quota_priv * priv = NULL;
-	struct timeval tv = {0, 0};
-
-	priv = this->private;
-	if (priv->min_free_disk_limit) {
-		gettimeofday (&tv, NULL);
-		if (tv.tv_sec > (priv->refresh_interval + 
-				 priv->min_disk_last_updated_time)) {
-			priv->min_disk_last_updated_time = tv.tv_sec;
-			gf_quota_update_current_free_disk (this);
-		}
-		if (priv->current_free_disk <= priv->min_free_disk_limit)
-			return -1;
-	}
+static inline uint64_t
+quota_time_elapsed (struct timeval *now, struct timeval *then)
+{
+        return (now->tv_sec - then->tv_sec);
+}
+
+
+int32_t
+quota_timeout (struct timeval *tv, int32_t timeout)
+{
+        struct timeval now       = {0,};
+        int32_t        timed_out = 0;
+
+        gettimeofday (&now, NULL);
+
+        if (quota_time_elapsed (&now, tv) >= timeout) {
+                timed_out = 1;
+        }
 
-	return 0;
+        return timed_out;
 }
 
 
-int
-quota_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, struct stat *buf)
+int32_t
+quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
+                   char *name, uuid_t par)
 {
-	struct quota_priv *priv = this->private;
-	struct quota_local *local = NULL;
+        int32_t               ret            = -1;
+        inode_t              *_inode         = NULL, *parent = NULL;
+        quota_inode_ctx_t    *ctx            = NULL;
+        quota_priv_t         *priv           = NULL;
+        quota_local_t        *local          = NULL;
+        char                  need_validate  = 0, need_unwind = 0;
+        int64_t               delta          = 0;
+        call_stub_t          *stub           = NULL;
+        int32_t               validate_count = 0, link_count = 0;
+        uint64_t              value          = 0;
+        char                  just_validated = 0;
+        uuid_t                trav_uuid      = {0,};
 
-	local = frame->local;
+        GF_VALIDATE_OR_GOTO ("quota", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, frame, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
-	if ((op_ret >= 0) && priv->disk_usage_limit) {
-		gf_quota_usage_subtract (this, (local->stbuf.st_blocks -
-						buf->st_blocks) * 512);
-		loc_wipe (&local->loc);
-	}
+        local = frame->local;
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+        delta = local->delta;
+
+        GF_VALIDATE_OR_GOTO (this->name, local->stub, out);
+
+        priv = this->private;
+
+        inode_ctx_get (inode, this, &value);
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+
+        _inode = inode_ref (inode);
+
+        LOCK (&local->lock);
+        {
+                just_validated = local->just_validated;
+                local->just_validated = 0;
+
+                if (just_validated) {
+                        local->validate_count--;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if ( par != NULL ) {
+                uuid_copy (trav_uuid, par);
+        }
+
+        do {
+                if (ctx != NULL) {
+                        LOCK (&ctx->lock);
+                        {
+                                if (ctx->limit >= 0) {
+                                        if (!just_validated
+                                            && quota_timeout (&ctx->tv,
+                                                              priv->timeout)) {
+                                                need_validate = 1;
+                                        } else if ((ctx->size + delta)
+                                                   >= ctx->limit) {
+                                                local->op_ret = -1;
+                                                local->op_errno = EDQUOT;
+                                                need_unwind = 1;
+                                        }
+                                }
+                        }
+                        UNLOCK (&ctx->lock);
+
+                        if (need_validate) {
+                                goto validate;
+                        }
+
+                        if (need_unwind) {
+                                break;
+                        }
+                }
+
+                if (__is_root_gfid (_inode->gfid)) {
+                        break;
+                }
+
+                parent = inode_parent (_inode, trav_uuid, name);
+
+                if (name != NULL) {
+                        name = NULL;
+                        uuid_clear (trav_uuid);
+                }
+
+                if (parent == NULL) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "cannot find parent for inode (gfid:%s), hence "
+                                "aborting enforcing quota-limits and continuing"
+                                " with the fop", uuid_utoa (_inode->gfid));
+                }
+
+                inode_unref (_inode);
+                _inode = parent;
+                just_validated = 0;
+
+                if (_inode == NULL) {
+                        break;
+                }
+
+                value = 0;
+                inode_ctx_get (_inode, this, &value);
+                ctx = (quota_inode_ctx_t *)(unsigned long)value;
+        } while (1);
+
+        ret = 0;
+
+        if (_inode != NULL) {
+                inode_unref (_inode);
+        }
+
+        LOCK (&local->lock);
+        {
+                validate_count = local->validate_count;
+                link_count = local->link_count;
+                if ((validate_count == 0) && (link_count == 0)) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+
+out:
+        return ret;
+
+validate:
+        LOCK (&local->lock);
+        {
+                loc_wipe (&local->validate_loc);
+
+                if (just_validated) {
+                        local->validate_count--;
+                }
 
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
+                local->validate_count++;
+                ret = quota_inode_loc_fill (_inode, &local->validate_loc);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cannot fill loc for inode (gfid:%s), hence "
+                                "aborting quota-checks and continuing with fop",
+                                uuid_utoa (_inode->gfid));
+                        local->validate_count--;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (ret < 0) {
+                goto loc_fill_failed;
+        }
+
+        STACK_WIND (frame, quota_validate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, &local->validate_loc,
+                    QUOTA_SIZE_KEY, NULL);
+
+loc_fill_failed:
+        inode_unref (_inode);
+        return 0;
 }
 
 
-int
-quota_truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int32_t op_ret, int32_t op_errno, struct stat *buf)
+int32_t
+quota_get_limit_value (inode_t *inode, xlator_t *this, int64_t *n)
 {
-	struct quota_local *local = NULL;
-	struct quota_priv  *priv = NULL;
+        int32_t       ret        = 0;
+        char         *path       = NULL;
+        limits_t     *limit_node = NULL;
+        quota_priv_t *priv       = NULL;
+
+        if (inode == NULL || n == NULL) {
+                ret = -1;
+                goto out;
+        }
 
-	priv = this->private;
-	local = frame->local;
+        *n = 0;
 
-	if (op_ret >= 0) {
-		local->stbuf = *buf;
-	}
+        ret = inode_path (inode, NULL, &path);
+        if (ret < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        priv = this->private;
+
+        list_for_each_entry (limit_node, &priv->limit_head, limit_list) {
+                if (strcmp (limit_node->path, path) == 0) {
+                        *n = limit_node->value;
+                        break;
+                }
+        }
 
-	STACK_WIND (frame, quota_truncate_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
-		    &local->loc, local->offset);
-	return 0;
+out:
+        GF_FREE (path);
+
+        return ret;
 }
 
 
-int
-quota_truncate (call_frame_t *frame, xlator_t *this,
-		loc_t *loc, off_t offset)
+static int32_t
+__quota_init_inode_ctx (inode_t *inode, int64_t limit, xlator_t *this,
+                        dict_t *dict, struct iatt *buf,
+                        quota_inode_ctx_t **context)
 {
-	struct quota_local *local = NULL;
-	struct quota_priv  *priv = NULL;
+        int32_t            ret  = -1;
+        int64_t           *size = 0;
+        quota_inode_ctx_t *ctx  = NULL;
 
-	priv = this->private;
+        if (inode == NULL) {
+                goto out;
+        }
 
-	if (priv->disk_usage_limit) {
-		local = CALLOC (1, sizeof (struct quota_local));
-		frame->local  = local;
+        QUOTA_ALLOC_OR_GOTO (ctx, quota_inode_ctx_t, out);
 
-		loc_copy (&local->loc, loc);
-		local->offset = offset;
+        ctx->limit = limit;
+        if (buf)
+                ctx->buf = *buf;
 
-		STACK_WIND (frame, quota_truncate_stat_cbk,
-			    FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->stat, loc);
-		return 0;
-	}
+        LOCK_INIT(&ctx->lock);
 
-	STACK_WIND (frame, quota_truncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->truncate,
-		    loc, offset);
-	return 0;
+        if (context != NULL) {
+                *context = ctx;
+        }
+
+        INIT_LIST_HEAD (&ctx->parents);
+
+        if (dict != NULL) {
+                ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+                if (ret == 0) {
+                        ctx->size = ntoh64 (*size);
+                        gettimeofday (&ctx->tv, NULL);
+                }
+        }
+
+        ret = __inode_ctx_put (inode, this, (uint64_t )(long)ctx);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "cannot set quota context in inode (gfid:%s)",
+                        uuid_utoa (inode->gfid));
+        }
+out:
+        return ret;
 }
 
 
-int
-quota_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, struct stat *buf)
+static int32_t
+quota_inode_ctx_get (inode_t *inode, int64_t limit, xlator_t *this,
+                     dict_t *dict, struct iatt *buf, quota_inode_ctx_t **ctx,
+                     char create_if_absent)
+{
+        int32_t  ret = 0;
+        uint64_t ctx_int;
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &ctx_int);
+
+                if ((ret == 0) && (ctx != NULL)) {
+                        *ctx = (quota_inode_ctx_t *) (unsigned long)ctx_int;
+                } else if (create_if_absent) {
+                        ret = __quota_init_inode_ctx (inode, limit, this, dict,
+                                                      buf, ctx);
+                }
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
+
+int32_t
+quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *dict, struct iatt *postparent)
 {
-	struct quota_priv  *priv = NULL;
-	struct quota_local *local = NULL;
+        int32_t            ret        = -1;
+        char               found      = 0;
+        quota_local_t     *local      = NULL;
+        quota_inode_ctx_t *ctx        = NULL;
+        quota_dentry_t    *dentry     = NULL;
+        int64_t           *size       = 0;
+        uint64_t           value      = 0;
+        limits_t          *limit_node = NULL;
+        quota_priv_t      *priv       = NULL;
+
+        local = frame->local;
+
+        priv = this->private;
+
+        inode_ctx_get (inode, this, &value);
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+
+        if ((op_ret < 0) || (local == NULL)
+            || (((ctx == NULL) || (ctx->limit == local->limit))
+                && (local->limit < 0) && !((IA_ISREG (buf->ia_type))
+                                           || (IA_ISLNK (buf->ia_type))))) {
+                goto unwind;
+        }
 
-	local = frame->local;
-	priv = this->private;
+        LOCK (&priv->lock);
+        {
+                list_for_each_entry (limit_node, &priv->limit_head,
+                                     limit_list) {
+                        if (strcmp (local->loc.path, limit_node->path) == 0) {
+                                uuid_copy (limit_node->gfid, buf->ia_gfid);
+                                break;
+                        }
+                }
+        }
+        UNLOCK (&priv->lock);
+
+        ret = quota_inode_ctx_get (local->loc.inode, local->limit, this, dict,
+                                   buf, &ctx, 1);
+        if ((ret == -1) || (ctx == NULL)) {
+                gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
+                        "context in inode(gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
 
-	if ((op_ret >= 0) && priv->disk_usage_limit) {
-		gf_quota_usage_subtract (this, (local->stbuf.st_blocks -
-						buf->st_blocks) * 512);
-		fd_unref (local->fd);
-	}
+        LOCK (&ctx->lock);
+        {
+
+                if (dict != NULL) {
+                        ret = dict_get_bin (dict, QUOTA_SIZE_KEY,
+                                            (void **) &size);
+                        if (ret == 0) {
+                                ctx->size = ntoh64 (*size);
+                                gettimeofday (&ctx->tv, NULL);
+                        }
+                }
+
+                if (local->limit != ctx->limit) {
+                        ctx->limit = local->limit;
+                }
 
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
+                ctx->buf = *buf;
+
+                if (!(IA_ISREG (buf->ia_type) || IA_ISLNK (buf->ia_type))) {
+                        goto unlock;
+                }
+
+                if (local->loc.name == NULL)
+                        goto unlock;
+
+                list_for_each_entry (dentry, &ctx->parents, next) {
+                        if ((strcmp (dentry->name, local->loc.name) == 0) &&
+                            (uuid_compare (local->loc.parent->gfid,
+                                           dentry->par) == 0)) {
+                                found = 1;
+                                break;
+                        }
+                }
+
+                if (!found) {
+                        dentry = __quota_dentry_new (ctx,
+                                                     (char *)local->loc.name,
+                                                     local->loc.parent->gfid);
+                        if (dentry == NULL) {
+                                /*
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "cannot create a new dentry (par:%"
+                                        PRId64", name:%s) for inode(ino:%"
+                                        PRId64", gfid:%s)",
+                                        uuid_utoa (local->loc.inode->gfid));
+                                */
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&ctx->lock);
+
+unwind:
+        QUOTA_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+                            dict, postparent);
+        return 0;
 }
 
 
-int
-quota_ftruncate_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno, struct stat *buf)
+int32_t
+quota_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xattr_req)
 {
-	struct quota_local *local = NULL;
-	struct quota_priv  *priv = NULL;
+        int32_t             ret         = -1;
+        int64_t             limit       = -1;
+        limits_t           *limit_node  = NULL;
+        gf_boolean_t        dict_newed  = _gf_false;
+        quota_priv_t       *priv        = NULL;
+        quota_local_t      *local       = NULL;
+
+        priv = this->private;
+
+        list_for_each_entry (limit_node, &priv->limit_head, limit_list) {
+                if (strcmp (limit_node->path, loc->path) == 0) {
+                        limit = limit_node->value;
+                }
+        }
 
-	priv = this->private;
-	local = frame->local;
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
 
-	if (op_ret >= 0) {
-		local->stbuf = *buf;
-	}
+        ret = loc_copy (&local->loc, loc);
+        if (ret == -1) {
+                goto err;
+        }
+
+        frame->local = local;
+
+        local->limit = limit;
+
+        if (limit < 0) {
+                goto wind;
+        }
+
+        if (xattr_req == NULL) {
+                xattr_req  = dict_new ();
+                dict_newed = _gf_true;
+        }
+
+        ret = dict_set_uint64 (xattr_req, QUOTA_SIZE_KEY, 0);
+        if (ret < 0) {
+                goto err;
+        }
+
+wind:
+        STACK_WIND (frame, quota_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+
+        ret = 0;
+
+err:
+        if (ret < 0) {
+                QUOTA_STACK_UNWIND (lookup, frame, -1, ENOMEM,
+                                    NULL, NULL, NULL, NULL);
+        }
+
+        if (dict_newed == _gf_true) {
+                dict_unref (xattr_req);
+        }
 
-	STACK_WIND (frame, quota_ftruncate_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
-		    local->fd, local->offset);
-	return 0;
+        return 0;
 }
 
 
-int
-quota_ftruncate (call_frame_t *frame, xlator_t *this,
-		 fd_t *fd, off_t offset)
+void
+quota_update_size (xlator_t *this, inode_t *inode, char *name, uuid_t par,
+                   int64_t delta)
 {
-	struct quota_local *local = NULL;
-	struct quota_priv  *priv = NULL;
+        inode_t           *_inode    = NULL;
+        inode_t           *parent    = NULL;
+        uint64_t           value     = 0;
+        quota_inode_ctx_t *ctx       = NULL;
+        uuid_t             trav_uuid = {0,};
+
+        GF_VALIDATE_OR_GOTO ("quota", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
+        inode_ctx_get (inode, this, &value);
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
 
-	priv = this->private;
+        _inode = inode_ref (inode);
 
-	if (priv->disk_usage_limit) {
-		local = CALLOC (1, sizeof (struct quota_local));
-		frame->local  = local;
+        if ( par != NULL ) {
+                uuid_copy (trav_uuid, par);
+        }
 
-		local->fd = fd_ref (fd);
-		local->offset = offset;
+        do {
+                if ((ctx != NULL) && (ctx->limit >= 0)) {
+                        LOCK (&ctx->lock);
+                        {
+                                ctx->size += delta;
+                        }
+                        UNLOCK (&ctx->lock);
+                }
 
-		STACK_WIND (frame, quota_ftruncate_fstat_cbk,
-			    FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->fstat, fd);
-		return 0;
-	}
+                if (__is_root_gfid (_inode->gfid)) {
+                        break;
+                }
+
+                parent = inode_parent (_inode, trav_uuid, name);
+                if (parent == NULL) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "cannot find parent for inode (gfid:%s), hence "
+                                "aborting size updation of parents",
+                                uuid_utoa (_inode->gfid));
+                }
+
+                if (name != NULL) {
+                        name = NULL;
+                        uuid_clear (trav_uuid);
+                }
+
+                inode_unref (_inode);
+                _inode = parent;
+
+                if (_inode == NULL) {
+                        break;
+                }
 
-	STACK_WIND (frame, quota_ftruncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd, offset);
-	return 0;
+                inode_ctx_get (_inode, this, &value);
+                ctx = (quota_inode_ctx_t *)(unsigned long)value;
+        } while (1);
+
+out:
+        return;
 }
 
 
-int
-quota_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int32_t op_ret, int32_t op_errno,
-		 inode_t *inode, struct stat *buf)
+int32_t
+quota_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-	struct quota_priv *priv = NULL;
+        int32_t                  ret            = 0;
+        uint64_t                 ctx_int        = 0;
+        quota_inode_ctx_t       *ctx            = NULL;
+        quota_local_t           *local          = NULL;
+        quota_dentry_t          *dentry         = NULL;
+        int64_t                  delta          = 0;
 
-	priv = this->private;
+        local = frame->local;
 
-	if ((op_ret >= 0) && priv->disk_usage_limit) {
-		gf_quota_usage_add (this, buf->st_blocks * 512);
-	}
+        if ((op_ret < 0) || (local == NULL)) {
+                goto out;
+        }
 
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
+        ret = inode_ctx_get (local->loc.inode, this, &ctx_int);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get the context", local->loc.path);
+                goto out;
+        }
+
+        ctx = (quota_inode_ctx_t *)(unsigned long) ctx_int;
+
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in %s (gfid:%s)",
+                        local->loc.path, uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *postbuf;
+        }
+        UNLOCK (&ctx->lock);
+
+        list_for_each_entry (dentry, &ctx->parents, next) {
+                delta = (postbuf->ia_blocks - prebuf->ia_blocks) * 512;
+                quota_update_size (this, local->loc.inode,
+                                   dentry->name, dentry->par, delta);
+        }
+
+out:
+        QUOTA_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+
+        return 0;
 }
 
 
-int
-quota_mknod (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, mode_t mode, dev_t rdev)
+int32_t
+quota_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     struct iovec *vector, int32_t count, off_t off,
+                     uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-	struct quota_priv *priv = NULL;
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
 
-	priv = this->private;
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
 
-	if (gf_quota_check_free_disk (this) == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"min-free-disk limit (%u) crossed, current available is %u",
-			priv->min_free_disk_limit, priv->current_free_disk);
-		STACK_UNWIND (frame, -1, ENOSPC, NULL, NULL);
-		return 0;
-	}
+        if (local->op_ret == -1) {
+                op_errno = local->op_errno;
+                goto unwind;
+        }
 
-        if (priv->current_disk_usage > priv->disk_usage_limit) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"Disk usage limit (%"PRIu64") crossed, current usage is %"PRIu64"",
-			priv->disk_usage_limit, priv->current_disk_usage);
-		STACK_UNWIND (frame, -1, ENOSPC, NULL, NULL);
-		return 0;
+        STACK_WIND (frame, quota_writev_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
+                    flags, iobref, xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int32_t count, off_t off,
+              uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        int32_t            ret     = -1, op_errno = EINVAL;
+        int32_t            parents = 0;
+        uint64_t           size    = 0;
+        quota_local_t     *local   = NULL;
+        quota_inode_ctx_t *ctx     = NULL;
+        quota_priv_t      *priv    = NULL;
+        call_stub_t       *stub    = NULL;
+        quota_dentry_t    *dentry  = NULL;
+
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO ("quota", this, unwind);
+        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+        local->loc.inode = inode_ref (fd->inode);
+
+        ret = quota_inode_ctx_get (fd->inode, -1, this, NULL, NULL, &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (fd->inode->gfid));
+                goto unwind;
+        }
+
+        stub = fop_writev_stub (frame, quota_writev_helper, fd, vector, count,
+                                off, flags, iobref, xdata);
+        if (stub == NULL) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        priv = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
+
+        size = iov_length (vector, count);
+        LOCK (&ctx->lock);
+        {
+                list_for_each_entry (dentry, &ctx->parents, next) {
+                        parents++;
+                }
+        }
+        UNLOCK (&ctx->lock);
+
+        local->delta = size;
+        local->stub = stub;
+        local->link_count = parents;
+
+        list_for_each_entry (dentry, &ctx->parents, next) {
+                ret = quota_check_limit (frame, fd->inode, this, dentry->name,
+                                         dentry->par);
+                if (ret == -1) {
+                        break;
+                }
+        }
+
+        stub = NULL;
+
+        LOCK (&local->lock);
+        {
+                local->link_count = 0;
+                if (local->validate_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
         }
 
-	STACK_WIND (frame, quota_mknod_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev);
-	return 0;
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
 }
 
 
-int
+int32_t
 quota_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int32_t op_ret, int32_t op_errno, inode_t *inode,
-		 struct stat *buf)
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-	struct quota_priv *priv = NULL;
+        QUOTA_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode,
+                            buf, preparent, postparent, xdata);
+        return 0;
+}
 
-	priv = this->private;
 
-	if ((op_ret >= 0) && priv->disk_usage_limit) {
-		gf_quota_usage_subtract (this, buf->st_blocks * 512);
-	}
+int32_t
+quota_mkdir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    mode_t mode, mode_t umask, dict_t *xdata)
+{
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
 
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
+        op_errno = local->op_errno;
+
+        if (local->op_ret == -1) {
+                goto unwind;
+        }
+
+        STACK_WIND (frame, quota_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL,
+                            NULL, NULL, NULL);
+        return 0;
 }
 
 
-int
-quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode)
+int32_t
+quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
 {
-	struct quota_priv *priv = NULL;
+        int32_t        ret            = 0, op_errno = 0;
+        quota_local_t *local          = NULL;
+        call_stub_t   *stub           = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	priv = this->private;
+        frame->local = local;
 
-	if (gf_quota_check_free_disk (this) == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"min-free-disk limit (%u) crossed, current available is %u",
-			priv->min_free_disk_limit, priv->current_free_disk);
-		STACK_UNWIND (frame, -1, ENOSPC, NULL, NULL);
-		return 0;
-		
-	}
+        local->link_count = 1;
+
+        ret = loc_copy (&local->loc, loc);
+        if (ret) {
+                op_errno = ENOMEM;
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto err;
+        }
+
+        stub = fop_mkdir_stub (frame, quota_mkdir_helper, loc, mode, umask,
+                               xdata);
+        if (stub == NULL) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->stub = stub;
+        local->delta = 0;
+
+        quota_check_limit (frame, loc->parent, this, NULL, NULL);
 
-        if (priv->current_disk_usage > priv->disk_usage_limit) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"Disk usage limit (%"PRIu64") crossed, current usage is %"PRIu64"",
-			priv->disk_usage_limit, priv->current_disk_usage);
-		STACK_UNWIND (frame, -1, ENOSPC, NULL, NULL);
-		return 0;
+        stub = NULL;
+
+        LOCK (&local->lock);
+        {
+                if (local->validate_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+
+                local->link_count = 0;
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
         }
 
-	STACK_WIND (frame, quota_mkdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode);
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                            NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
-int
+int32_t
+quota_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        int32_t            ret    = -1;
+        quota_local_t     *local  = NULL;
+        quota_inode_ctx_t *ctx    = NULL;
+        quota_dentry_t    *dentry = NULL;
+
+        local = frame->local;
+        if (op_ret < 0) {
+                goto unwind;
+        }
+
+        ret = quota_inode_ctx_get (inode, -1, this, NULL, buf, &ctx, 1);
+        if ((ret == -1) || (ctx == NULL)) {
+                gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
+                        "context in inode(gfid:%s)",
+                        uuid_utoa (inode->gfid));
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *buf;
+
+                dentry = __quota_dentry_new (ctx, (char *)local->loc.name,
+                                             local->loc.parent->gfid);
+                if (dentry == NULL) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cannot create a new dentry (name:%s) for "
+                                "inode(gfid:%s)", local->loc.name,
+                                uuid_utoa (local->loc.inode->gfid));
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unlock;
+                }
+        }
+unlock:
+        UNLOCK (&ctx->lock);
+
+unwind:
+        QUOTA_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
+                            preparent, postparent, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_create_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                     dict_t *xdata)
+{
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
+
+        if (local->op_ret == -1) {
+                op_errno = local->op_errno;
+                goto unwind;
+        }
+
+        STACK_WIND (frame, quota_create_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL,
+                            NULL, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int32_t            ret            = -1;
+        quota_local_t     *local          = NULL;
+        call_stub_t       *stub           = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
+
+        frame->local = local;
+
+        ret = loc_copy (&local->loc, loc);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto err;
+        }
+
+        stub = fop_create_stub (frame, quota_create_helper, loc, flags, mode,
+                                umask, fd, xdata);
+        if (stub == NULL) {
+                goto err;
+        }
+
+        local->link_count = 1;
+        local->stub = stub;
+        local->delta = 0;
+
+        quota_check_limit (frame, loc->parent, this, NULL, NULL);
+
+        stub = NULL;
+
+        LOCK (&local->lock);
+        {
+                local->link_count = 0;
+                if (local->validate_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
+
+        return 0;
+}
+
+
+int32_t
 quota_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-	struct quota_local *local = NULL;
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+        uint64_t           value = 0;
 
-	local = frame->local;
+        if (op_ret < 0) {
+                goto out;
+        }
 
-	if (local) {
-		if (op_ret >= 0) {
-			gf_quota_usage_subtract (this,
-						 local->stbuf.st_blocks * 512);
-		}
-		loc_wipe (&local->loc);
-	}
+        local = (quota_local_t *) frame->local;
+
+        inode_ctx_get (local->loc.inode, this, &value);
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
 
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+        quota_update_size (this, local->loc.inode, (char *)local->loc.name,
+                           local->loc.parent->gfid,
+                           (-(ctx->buf.ia_blocks * 512)));
+
+out:
+        QUOTA_STACK_UNWIND (unlink, frame, op_ret, op_errno, preparent,
+                            postparent, xdata);
+        return 0;
 }
 
 
-int
-quota_unlink_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, struct stat *buf)
+int32_t
+quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
-	struct quota_local *local = NULL;
+        int32_t        ret = 0;
+        quota_local_t *local = NULL;
 
-	local = frame->local;
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
 
-	if (op_ret >= 0) {
-		if (buf->st_nlink == 1) {
-			local->stbuf = *buf;
-		}
-	}
+        frame->local = local;
+
+        ret = loc_copy (&local->loc, loc);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto err;
+        }
 
-	STACK_WIND (frame, quota_unlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->unlink,
-		    &local->loc);
+        STACK_WIND (frame, quota_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
 
-	return 0;
+        ret = 0;
+
+err:
+        if (ret == -1) {
+                QUOTA_STACK_UNWIND (unlink, frame, -1, 0, NULL, NULL, NULL);
+        }
+
+        return 0;
 }
 
 
-int
-quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+int32_t
+quota_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        int32_t               ret          = -1;
+        quota_local_t        *local        = NULL;
+        quota_inode_ctx_t    *ctx          = NULL;
+        quota_dentry_t       *dentry       = NULL;
+        char                  found        = 0;
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = (quota_local_t *) frame->local;
+
+        quota_update_size (this, local->loc.parent, NULL, NULL,
+                           (buf->ia_blocks * 512));
+
+        ret = quota_inode_ctx_get (inode, -1, this, NULL, NULL, &ctx, 0);
+        if ((ret == -1) || (ctx == NULL)) {
+                gf_log (this->name, GF_LOG_WARNING, "cannot find quota "
+                        "context in %s (gfid:%s)", local->loc.path,
+                        uuid_utoa (inode->gfid));
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                list_for_each_entry (dentry, &ctx->parents, next) {
+                        if ((strcmp (dentry->name, local->loc.name) == 0) &&
+                            (uuid_compare (local->loc.parent->gfid,
+                                           dentry->par) == 0)) {
+                                found = 1;
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "new entry being linked (name:%s) for "
+                                        "inode (gfid:%s) is already present "
+                                        "in inode-dentry-list", dentry->name,
+                                        uuid_utoa (local->loc.inode->gfid));
+                                break;
+                        }
+                }
+
+                if (!found) {
+                        dentry = __quota_dentry_new (ctx,
+                                                     (char *)local->loc.name,
+                                                     local->loc.parent->gfid);
+                        if (dentry == NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "cannot create a new dentry (name:%s) "
+                                        "for inode(gfid:%s)", local->loc.name,
+                                        uuid_utoa (local->loc.inode->gfid));
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+
+                ctx->buf = *buf;
+        }
+unlock:
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
+
+        return 0;
+}
+
+
+int32_t
+quota_link_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                   loc_t *newloc, dict_t *xdata)
 {
-	struct quota_local *local = NULL;
-	struct quota_priv  *priv = NULL;
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
 
-	priv = this->private;
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
 
-	if (priv->disk_usage_limit) {
-		local = CALLOC (1, sizeof (struct quota_local));
-		frame->local = local;
+        op_errno = local->op_errno;
 
-		loc_copy (&local->loc, loc);
+        if (local->op_ret == -1) {
+                goto unwind;
+        }
 
-		STACK_WIND (frame,
-			    quota_unlink_stat_cbk,
-			    FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->stat,
-			    loc);
-		return 0;
-	}
+        STACK_WIND (frame, quota_link_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+        return 0;
 
-	STACK_WIND (frame, quota_unlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->unlink,
-		    loc);
-	return 0;
+unwind:
+        QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
+                            NULL, NULL, NULL);
+        return 0;
 }
 
 
-int
-quota_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int32_t op_ret, int32_t op_errno)
+int32_t
+quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-	struct quota_local *local = NULL;
+        int32_t            ret   = -1, op_errno = ENOMEM;
+        quota_local_t     *local = NULL;
+        call_stub_t       *stub  = NULL;
+        quota_inode_ctx_t *ctx = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
 
-	local = frame->local;
+        frame->local = (void *) local;
 
-	if (local) {
-		if (op_ret >= 0) {
-			gf_quota_usage_subtract (this, local->stbuf.st_blocks * 512);
-		}
-		loc_wipe (&local->loc);
-	}
+        ret = loc_copy (&local->loc, newloc);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto err;
+        }
+
+        stub = fop_link_stub (frame, quota_link_helper, oldloc, newloc, xdata);
+        if (stub == NULL) {
+                goto err;
+        }
+
+        local->link_count = 1;
+        local->stub = stub;
+
+        ret = quota_inode_ctx_get (oldloc->inode, -1, this, NULL, NULL, &ctx,
+                                   0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        oldloc->inode ? uuid_utoa (oldloc->inode->gfid) : "0");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->delta = ctx->buf.ia_blocks * 512;
 
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+        quota_check_limit (frame, newloc->parent, this, NULL, NULL);
+
+        stub = NULL;
+
+        LOCK (&local->lock);
+        {
+                if (local->validate_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+
+                local->link_count = 0;
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+
+        ret = 0;
+err:
+        if (ret < 0) {
+                QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
+                                    NULL, NULL, NULL);
+        }
+
+        return 0;
 }
 
 
-int
-quota_rmdir_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct stat *buf)
+int32_t
+quota_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  struct iatt *preoldparent, struct iatt *postoldparent,
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
 {
-	struct quota_local *local = NULL;
+        int32_t               ret              = -1;
+        quota_local_t        *local            = NULL;
+        quota_inode_ctx_t    *ctx              = NULL;
+        quota_dentry_t       *old_dentry       = NULL, *dentry = NULL;
+        char                  new_dentry_found = 0;
+        int64_t               size             = 0;
+
+        if (op_ret < 0) {
+                goto out;
+        }
 
-	local = frame->local;
+        local = frame->local;
+        if (local == NULL) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
 
-	if (op_ret >= 0) {
-		local->stbuf = *buf;
-	}
+        if (IA_ISREG (local->oldloc.inode->ia_type)
+            || IA_ISLNK (local->oldloc.inode->ia_type)) {
+                size = buf->ia_blocks * 512;
+        }
+
+        if (local->oldloc.parent != local->newloc.parent) {
+                quota_update_size (this, local->oldloc.parent, NULL, NULL, (-size));
+                quota_update_size (this, local->newloc.parent, NULL, NULL, size);
+        }
+
+        if (!(IA_ISREG (local->oldloc.inode->ia_type)
+              || IA_ISLNK (local->oldloc.inode->ia_type))) {
+                goto out;
+        }
+
+        ret = quota_inode_ctx_get (local->oldloc.inode, -1, this, NULL, NULL,
+                                   &ctx, 0);
+        if ((ret == -1) || (ctx == NULL)) {
+                gf_log (this->name, GF_LOG_WARNING, "quota context not"
+                        "set in inode(gfid:%s)",
+                        uuid_utoa (local->oldloc.inode->gfid));
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                /* decision of whether to create a context in newloc->inode
+                 * depends on fuse_rename_cbk's choice of inode it retains
+                 * after rename. currently it just associates oldloc->inode
+                 * with new parent and name. If this changes, following code
+                 * should be changed to set a new context in newloc->inode.
+                 */
+                list_for_each_entry (dentry, &ctx->parents, next) {
+                        if ((strcmp (dentry->name, local->oldloc.name) == 0) &&
+                            (uuid_compare (local->oldloc.parent->gfid,
+                                           dentry->par) == 0)) {
+                                old_dentry = dentry;
+                        } else if ((strcmp (dentry->name,
+                                            local->newloc.name) == 0) &&
+                                   (uuid_compare (local->oldloc.parent->gfid,
+                                                  dentry->par) == 0)) {
+                                new_dentry_found = 1;
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "new entry being linked (name:%s) for "
+                                        "inode (gfid:%s) is already present "
+                                        "in inode-dentry-list", dentry->name,
+                                        uuid_utoa (local->newloc.inode->gfid));
+                                break;
+                        }
+                }
+
+                if (old_dentry != NULL) {
+                        __quota_dentry_free (old_dentry);
+                } else {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "dentry corresponding to the path just renamed "
+                                "(name:%s) is not present", local->oldloc.name);
+                }
+
+                if (!new_dentry_found) {
+                        dentry = __quota_dentry_new (ctx,
+                                                     (char *)local->newloc.name,
+                                                     local->newloc.parent->gfid);
+                        if (dentry == NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "cannot create a new dentry (name:%s) "
+                                        "for inode(gfid:%s)", local->newloc.name,
+                                        uuid_utoa (local->newloc.inode->gfid));
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+
+                ctx->buf = *buf;
+        }
+unlock:
+        UNLOCK (&ctx->lock);
 
-	STACK_WIND (frame, quota_rmdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rmdir,
-		    &local->loc);
+out:
+        QUOTA_STACK_UNWIND (rename, frame, op_ret, op_errno, buf, preoldparent,
+                            postoldparent, prenewparent, postnewparent, xdata);
 
-	return 0;
+        return 0;
 }
 
 
-int
-quota_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc)
+int32_t
+quota_rename_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                     loc_t *newloc, dict_t *xdata)
 {
-	struct quota_local *local = NULL;
-	struct quota_priv  *priv = NULL;
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
 
-	priv = this->private;
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
 
-	if (priv->disk_usage_limit) {
-		local = CALLOC (1, sizeof (struct quota_local));
-		frame->local = local;
+        op_errno = local->op_errno;
 
-		loc_copy (&local->loc, loc);
+        if (local->op_ret == -1) {
+                goto unwind;
+        }
 
-		STACK_WIND (frame, quota_rmdir_stat_cbk,
-			    FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->stat, loc);
-		return 0;
-	}
+        STACK_WIND (frame, quota_rename_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+        return 0;
 
-	STACK_WIND (frame, quota_rmdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rmdir,
-		    loc);
-	return 0;
+unwind:
+        QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL,
+                            NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 
-int
+int32_t
+quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+              loc_t *newloc, dict_t *xdata)
+{
+        int32_t            ret            = -1, op_errno = ENOMEM;
+        quota_local_t     *local          = NULL;
+        call_stub_t       *stub           = NULL;
+        quota_inode_ctx_t *ctx            = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
+
+        frame->local = local;
+
+        ret = loc_copy (&local->oldloc, oldloc);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto err;
+        }
+
+        ret = loc_copy (&local->newloc, newloc);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto err;
+        }
+
+        stub = fop_rename_stub (frame, quota_rename_helper, oldloc, newloc,
+                                xdata);
+        if (stub == NULL) {
+                goto err;
+        }
+
+        local->link_count = 1;
+        local->stub = stub;
+
+        if (IA_ISREG (oldloc->inode->ia_type)
+            || IA_ISLNK (oldloc->inode->ia_type)) {
+                ret = quota_inode_ctx_get (oldloc->inode, -1, this, NULL, NULL,
+                                           &ctx, 0);
+                if (ctx == NULL) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "quota context not set in inode (gfid:%s)",
+                                oldloc->inode ? uuid_utoa (oldloc->inode->gfid)
+                                : "0");
+                        op_errno = EINVAL;
+                        goto err;
+                }
+                local->delta = ctx->buf.ia_blocks * 512;
+        } else {
+                local->delta = 0;
+        }
+
+        quota_check_limit (frame, newloc->parent, this, NULL, NULL);
+
+        stub = NULL;
+
+        LOCK (&local->lock);
+        {
+                if (local->validate_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+
+                local->link_count = 0;
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+
+        ret = 0;
+err:
+        if (ret == -1) {
+                QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
+                                    NULL, NULL, NULL, NULL, NULL);
+        }
+
+        return 0;
+}
+
+
+int32_t
 quota_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, inode_t *inode,
-		   struct stat *buf)
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	struct quota_priv *priv = NULL;
+        int64_t            size   = 0;
+        quota_local_t     *local  = NULL;
+        quota_inode_ctx_t *ctx    = NULL;
+        quota_dentry_t    *dentry = NULL;
+
+        if (op_ret < 0) {
+                goto out;
+        }
 
-	priv = this->private;
+        local = frame->local;
+        size = buf->ia_blocks * 512;
 
-	if ((op_ret >= 0) && priv->disk_usage_limit) {
-		gf_quota_usage_add (this, buf->st_blocks * 512);
-	}
+        quota_update_size (this, local->loc.parent, NULL, NULL, size);
 
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 1);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *buf;
+
+                dentry = __quota_dentry_new (ctx, (char *)local->loc.name,
+                                             local->loc.parent->gfid);
+                if (dentry == NULL) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cannot create a new dentry (name:%s) for "
+                                "inode(gfid:%s)", local->loc.name,
+                                uuid_utoa (local->loc.inode->gfid));
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                }
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
+
+        return 0;
 }
 
 
 int
-quota_symlink (call_frame_t *frame, xlator_t *this,
-	       const char *linkpath, loc_t *loc)
+quota_symlink_helper (call_frame_t *frame, xlator_t *this, const char *linkpath,
+                      loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	struct quota_priv *priv = NULL;
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
 
-	priv = this->private;
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
 
-	if (gf_quota_check_free_disk (this) == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"min-free-disk limit (%u) crossed, current available is %u",
-			priv->min_free_disk_limit, priv->current_free_disk);
-		STACK_UNWIND (frame, -1, ENOSPC, NULL, NULL);
-		return 0;
-		
-	}
-        if (priv->current_disk_usage > priv->disk_usage_limit) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"Disk usage limit (%"PRIu64") crossed, current usage is %"PRIu64"",
-			priv->disk_usage_limit, priv->current_disk_usage);
-		STACK_UNWIND (frame, -1, ENOSPC, NULL, NULL);
-		return 0;
+        if (local->op_ret == -1) {
+                op_errno = local->op_errno;
+                goto unwind;
         }
 
-	STACK_WIND (frame, quota_symlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc);
-	return 0;
+        STACK_WIND (frame, quota_symlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                    xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL,
+                            NULL, NULL, NULL);
+        return 0;
 }
 
 
 int
-quota_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno,
-		  fd_t *fd, inode_t *inode, struct stat *buf)
+quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	struct quota_priv *priv = this->private;
-	int                ret = 0;
+        int32_t          ret      = -1;
+        int32_t          op_errno = ENOMEM;
+        quota_local_t   *local    = NULL;
+        call_stub_t     *stub     = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
 
-	if ((op_ret >= 0) && priv->disk_usage_limit) {
-		gf_quota_usage_add (this, buf->st_blocks * 512);
+        frame->local = local;
 
-		ret = fd_ctx_set (fd, this, 1);
-	}
+        ret = loc_copy (&local->loc, loc);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto err;
+        }
 
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
-	return 0;
+        local->link_count = 1;
+
+        stub = fop_symlink_stub (frame, quota_symlink_helper, linkpath, loc,
+                                 umask, xdata);
+        if (stub == NULL) {
+                goto err;
+        }
+
+        local->stub = stub;
+        local->delta = strlen (linkpath);
+
+        quota_check_limit (frame, loc->parent, this, NULL, NULL);
+
+        stub = NULL;
+
+        LOCK (&local->lock);
+        {
+                if (local->validate_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+
+                local->link_count = 0;
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+
+        return 0;
+
+err:
+        QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
+                            NULL, NULL);
+
+        return 0;
 }
 
 
-int
-quota_create (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags, mode_t mode, fd_t *fd)
+int32_t
+quota_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-	struct quota_priv *priv = NULL;
+        quota_local_t     *local = NULL;
+        int64_t            delta = 0;
+        quota_inode_ctx_t *ctx   = NULL;
 
-	priv = this->private;
+        if (op_ret < 0) {
+                goto out;
+        }
 
-	if (gf_quota_check_free_disk (this) == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"min-free-disk limit (%u) crossed, current available is %u",
-			priv->min_free_disk_limit, priv->current_free_disk);
-		STACK_UNWIND (frame, -1, ENOSPC, NULL, NULL, NULL);
-		return 0;
-		
-	}
-        if (priv->current_disk_usage > priv->disk_usage_limit) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"Disk usage limit (%"PRIu64") crossed, current usage is %"PRIu64"",
-			priv->disk_usage_limit, priv->current_disk_usage);
-		STACK_UNWIND (frame, -1, ENOSPC, NULL, NULL, NULL);
-		return 0;
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
+
+        delta = (postbuf->ia_blocks - prebuf->ia_blocks) * 512;
+
+        quota_update_size (this, local->loc.inode, NULL, NULL, delta);
+
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *postbuf;
         }
+        UNLOCK (&ctx->lock);
 
-	STACK_WIND (frame, quota_create_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, fd);
-	return 0;
+out:
+        QUOTA_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
+                            postbuf, xdata);
+        return 0;
 }
 
 
-int
-quota_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int32_t op_errno, fd_t *fd)
+int32_t
+quota_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                dict_t *xdata)
 {
-	int                ret = 0;
+        int32_t          ret   = -1;
+        quota_local_t   *local = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
 
-	if (op_ret >= 0)
-		ret = fd_ctx_set (fd, this, 1);
+        frame->local = local;
 
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
+        ret =  loc_copy (&local->loc, loc);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto err;
+        }
+
+        STACK_WIND (frame, quota_truncate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
 }
 
 
-int
-quota_open (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t flags, fd_t *fd)
+int32_t
+quota_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        quota_local_t     *local = NULL;
+        int64_t            delta = 0;
+        quota_inode_ctx_t *ctx   = NULL;
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
+
+        delta = (postbuf->ia_blocks - prebuf->ia_blocks) * 512;
+
+        quota_update_size (this, local->loc.inode, NULL, NULL, delta);
+
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *postbuf;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf,
+                            postbuf, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 dict_t *xdata)
 {
-	STACK_WIND (frame, quota_open_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd);
-	return 0;
+        quota_local_t   *local = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL)
+                goto err;
+
+        frame->local = local;
+
+        local->loc.inode = inode_ref (fd->inode);
+
+        STACK_WIND (frame, quota_ftruncate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+        return 0;
 }
 
 
-int
-quota_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, struct stat *stbuf)
+int32_t
+quota_send_dir_limit_to_cli (call_frame_t *frame, xlator_t *this,
+                             inode_t *inode, const char *name)
 {
-	struct quota_priv *priv = NULL;
-	struct quota_local *local = NULL;
+        int32_t            ret               = 0;
+        char               dir_limit [1024]  = {0, };
+        dict_t            *dict              = NULL;
+        quota_inode_ctx_t *ctx               = NULL;
+        uint64_t           value             = 0;
+
+        ret = inode_ctx_get (inode, this, &value);
+        if (ret < 0)
+                goto out;
+
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+        snprintf (dir_limit, 1024, "%"PRId64",%"PRId64, ctx->size, ctx->limit);
+
+        dict = dict_new ();
+        if (dict == NULL) {
+                ret = -1;
+                goto out;
+        }
 
+        ret = dict_set_str (dict, (char *) name, dir_limit);
+        if (ret < 0)
+                goto out;
 
-	priv = this->private;
-	local = frame->local;
+        gf_log (this->name, GF_LOG_INFO, "str = %s", dir_limit);
 
-	if (priv->disk_usage_limit) {
-		if (op_ret >= 0) { 
-			gf_quota_usage_add (this, (stbuf->st_blocks -
-						   local->stbuf.st_blocks) * 512);
-		}
-		fd_unref (local->fd);
-		dict_unref (local->refs);
-	}
+        QUOTA_STACK_UNWIND (getxattr, frame, 0, 0, dict, NULL);
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+int32_t
+quota_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+        int32_t ret     = 0;
+
+        if (name && strcasecmp (name, "trusted.limit.list") == 0) {
+                ret = quota_send_dir_limit_to_cli (frame, this, fd->inode,
+                                                   name);
+                if (ret == 0) {
+                        return 0;
+                }
+        }
+
+        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        int32_t ret     = 0;
+
+        if ((name != NULL) && strcasecmp (name, "trusted.limit.list") == 0) {
+                ret = quota_send_dir_limit_to_cli (frame, this, loc->inode,
+                                                   name);
+                if (ret == 0)
+                        return 0;
+        }
+
+        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
+
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                if (buf)
+                        ctx->buf = *buf;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+        int32_t        ret   = -1;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+        ret = loc_copy (&local->loc, loc);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto unwind;
+        }
+
+        STACK_WIND (frame, quota_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (stat, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+quota_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 dict_t *xdata)
+{
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
+
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                if (buf)
+                        ctx->buf = *buf;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+
+        local->loc.inode = inode_ref (fd->inode);
+
+        STACK_WIND (frame, quota_fstat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (fstat, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+quota_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, const char *path,
+                    struct iatt *buf, dict_t *xdata)
+{
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
+
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *buf;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, buf, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+        int32_t        ret   = -1;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+
+        ret = loc_copy (&local->loc, loc);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto unwind;
+        }
+
+        STACK_WIND (frame, quota_readlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+quota_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                 int32_t count, struct iatt *buf, struct iobref *iobref,
+                 dict_t *xdata)
+{
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
+
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *buf;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
+                            buf, iobref, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, uint32_t flags, dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+
+        local->loc.inode = inode_ref (fd->inode);
+
+        STACK_WIND (frame, quota_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                    xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, -1, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+quota_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
+
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *postbuf;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+        return 0;
+}
+
+
+int32_t
+quota_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        local->loc.inode = inode_ref (fd->inode);
+
+        frame->local = local;
+
+        STACK_WIND (frame, quota_fsync_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+
+}
+
+
+int32_t
+quota_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
+{
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
+
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                if (statpost)
+                        ctx->buf = *statpost;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
+                            statpost, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+        int32_t        ret   = -1;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+
+        ret = loc_copy (&local->loc, loc);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto unwind;
+        }
+
+        STACK_WIND (frame, quota_setattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+quota_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
+{
+        quota_local_t     *local = NULL;
+        quota_inode_ctx_t *ctx   = NULL;
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto out;
+        }
+
+        quota_inode_ctx_get (local->loc.inode, -1, this, NULL, NULL,
+                             &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *statpost;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        QUOTA_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, statpre,
+                            statpost, xdata);
+        return 0;
+}
+
+
+int32_t
+quota_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+
+        local->loc.inode = inode_ref (fd->inode);
+
+        STACK_WIND (frame, quota_fsetattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+quota_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        int32_t            ret    = -1;
+        quota_local_t     *local  = NULL;
+        quota_inode_ctx_t *ctx    = NULL;
+        quota_dentry_t    *dentry = NULL;
+
+        local = frame->local;
+        if (op_ret < 0) {
+                goto unwind;
+        }
+
+        ret = quota_inode_ctx_get (inode, -1, this, NULL, buf, &ctx, 1);
+        if ((ret == -1) || (ctx == NULL)) {
+                gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
+                        "context in inode (gfid:%s)", uuid_utoa (inode->gfid));
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *buf;
+
+                dentry = __quota_dentry_new (ctx, (char *)local->loc.name,
+                                             local->loc.parent->gfid);
+                if (dentry == NULL) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cannot create a new dentry (name:%s) for "
+                                "inode(gfid:%s)", local->loc.name,
+                                uuid_utoa (local->loc.inode->gfid));
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unlock;
+                }
+        }
+unlock:
+        UNLOCK (&ctx->lock);
 
-	STACK_UNWIND (frame, op_ret, op_errno, stbuf);
-	return 0;
+unwind:
+        QUOTA_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode,
+                            buf, preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
-quota_writev_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret,	int32_t op_errno, struct stat *buf)
+quota_mknod_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
-	struct quota_local *local = NULL;
-	struct quota_priv  *priv = NULL;
-	int                 iovlen = 0;
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
 
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
 
-	local = frame->local;
-	priv = this->private;
+        if (local->op_ret == -1) {
+                op_errno = local->op_errno;
+                goto unwind;
+        }
 
-	if (op_ret >= 0) {
-		if (priv->current_disk_usage > priv->disk_usage_limit) {
-			iovlen = iov_length (local->vector, local->count);
+        STACK_WIND (frame, quota_mknod_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                    xdata);
 
-			if (iovlen > (buf->st_blksize - (buf->st_size % buf->st_blksize))) {
-				fd_unref (local->fd);
-				dict_unref (local->refs);
-				STACK_UNWIND (frame, -1, ENOSPC, NULL);
-				return 0;
-			}
-		}
-		local->stbuf = *buf;
-	}
-	
-	STACK_WIND (frame, quota_writev_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->writev,
-		    local->fd, local->vector, local->count, local->offset);
+        return 0;
 
-	return 0;
+unwind:
+        QUOTA_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL,
+                            NULL, NULL, NULL);
+        return 0;
 }
 
 
 int
-quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	      struct iovec *vector, int32_t count, off_t off)
+quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *xdata)
 {
-	struct quota_local *local = NULL;
-	struct quota_priv  *priv = NULL;
+        int32_t            ret            = -1;
+        quota_local_t     *local          = NULL;
+        call_stub_t       *stub           = NULL;
 
-	priv = this->private;
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
 
-	if (gf_quota_check_free_disk (this) == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"min-free-disk limit (%u) crossed, current available is %u",
-			priv->min_free_disk_limit, priv->current_free_disk);
-		STACK_UNWIND (frame, -1, ENOSPC, NULL);
-		return 0;
-	}
+        frame->local = local;
 
-	if (priv->disk_usage_limit) {
-		local = CALLOC (1, sizeof (struct quota_local));
-		local->fd     = fd_ref (fd);
-		local->refs   = dict_ref (frame->root->req_refs);
-		local->vector = vector;
-		local->count  = count;
-		local->offset = off;
-		frame->local  = local;
-
-		STACK_WIND (frame, quota_writev_fstat_cbk,
-			    FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->fstat, fd);
-		return 0;
-	}
+        ret = loc_copy (&local->loc, loc);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
+                goto err;
+        }
 
-	STACK_WIND (frame, quota_writev_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->writev,
-		    fd, vector, count, off);
-	return 0;
+        stub = fop_mknod_stub (frame, quota_mknod_helper, loc, mode, rdev,
+                               umask, xdata);
+        if (stub == NULL) {
+                goto err;
+        }
+
+        local->link_count = 1;
+        local->stub = stub;
+        local->delta = 0;
+
+        quota_check_limit (frame, loc->parent, this, NULL, NULL);
+
+        stub = NULL;
+
+        LOCK (&local->lock);
+        {
+                local->link_count = 0;
+                if (local->validate_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+
+        return 0;
 }
 
+int
+quota_setxattr_cbk (call_frame_t *frame, void *cookie,
+                    xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
+{
+        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
 
 int
-quota_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+quota_setxattr (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
 {
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"failed to remove the disk-usage value: %s",
-			strerror (op_errno));
-	} 
-	
-	STACK_DESTROY (frame->root);
-	return 0;
+        int             op_errno = EINVAL;
+        int             op_ret   = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
+                                   op_errno, err);
+
+        STACK_WIND (frame, quota_setxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
+        return 0;
 }
 
+int
+quota_fsetxattr_cbk (call_frame_t *frame, void *cookie,
+                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
+{
+        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
 
 int
-quota_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
+quota_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 dict_t *dict, int flags, dict_t *xdata)
 {
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"failed to set the disk-usage value: %s",
-			strerror (op_errno));
-	} 
+        int32_t         op_ret   = -1;
+        int32_t         op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
+                                   op_errno, err);
+
+        STACK_WIND (frame, quota_fsetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
+ err:
+        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
+        return 0;
+}
+
+
+int
+quota_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        QUOTA_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+quota_removexattr (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, const char *name, dict_t *xdata)
+{
+        int32_t         op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (this, err);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.quota*",
+                                 name, op_errno, err);
 
-	STACK_DESTROY (frame->root);
-	return 0;
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        STACK_WIND (frame, quota_removexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+err:
+        QUOTA_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
+        return 0;
 }
 
 
 int
-quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, struct statvfs *statvfs)
+quota_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	struct quota_priv *priv = NULL;
-	uint64_t           f_blocks = 0;
-	int64_t            f_bfree = 0;
-	uint64_t           f_bused = 0;
+        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
 
+int
+quota_fremovexattr (call_frame_t *frame, xlator_t *this,
+                    fd_t *fd, const char *name, dict_t *xdata)
+{
+        int32_t         op_ret   = -1;
+        int32_t         op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.quota*",
+                                 name, op_errno, err);
+
+        STACK_WIND (frame, quota_fremovexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+ err:
+        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
+        return 0;
+}
 
-	priv = this->private;
 
-	if (op_ret != 0)
+int32_t
+quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                  dict_t *xdata)
+{
+	inode_t           *root_inode = NULL;
+        quota_priv_t      *priv       = NULL;
+        uint64_t           value      = 0;
+        quota_inode_ctx_t *ctx        = NULL;
+        limits_t          *limit_node = NULL;
+	int64_t            usage      = -1;
+	int64_t            avail      = -1;
+        int64_t            blocks     = 0;
+
+	root_inode = cookie;
+
+        /* This fop will fail mostly in case of client disconnect's,
+         * which is already logged. Hence, not logging here */
+        if (op_ret == -1)
+                goto unwind;
+	/*
+	 * We should never get here unless quota_statfs (below) sent us a
+	 * cookie, and it would only do so if the value was non-NULL.  This
+	 * check is therefore just routine defensive coding.
+	 */
+	if (!root_inode) {
+		gf_log(this->name,GF_LOG_WARNING,
+		       "null inode, cannot adjust for quota");
+		goto unwind;
+	}
+	if (!root_inode->table || (root_inode != root_inode->table->root)) {
+		gf_log(this->name,GF_LOG_WARNING,
+		       "non-root inode, cannot adjust for quota");
 		goto unwind;
+	}
 
-	f_blocks = priv->disk_usage_limit / statvfs->f_frsize;
-	f_bused = priv->current_disk_usage / statvfs->f_frsize;
+        inode_ctx_get (root_inode, this, &value);
+	if (!value) {
+		goto unwind;
+	}
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+	usage = (ctx->size) / buf->f_bsize;
+        priv = this->private;
+
+        list_for_each_entry (limit_node, &priv->limit_head, limit_list) {
+		/* Notice that this only works for volume-level quota. */
+                if (strcmp (limit_node->path, "/") == 0) {
+                        blocks = limit_node->value / buf->f_bsize;
+                        if (usage > blocks) {
+                                break;
+                        }
+
+			buf->f_blocks = blocks;
+			avail = buf->f_blocks - usage;
+			if (buf->f_bfree > avail) {
+				buf->f_bfree = avail;
+			}
+			/*
+			 * We have to assume that the total assigned quota
+			 * won't cause us to dip into the reserved space,
+			 * because dealing with the overcommitted cases is
+			 * just too hairy (especially when different bricks
+			 * might be using different reserved percentages and
+			 * such).
+			 */
+			buf->f_bavail = buf->f_bfree;
+			break;
+                }
+        }
 
-	if (f_blocks && (f_blocks < statvfs->f_blocks))
-		statvfs->f_blocks = f_blocks;
+unwind:
+	if (root_inode) {
+		inode_unref(root_inode);
+	}
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
 
-	f_bfree = (statvfs->f_blocks - f_bused);
 
-	if (f_bfree >= 0)
-		statvfs->f_bfree = statvfs->f_bavail = f_bfree;
-	else
-		statvfs->f_bfree = statvfs->f_bavail = 0;
+int32_t
+quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+	inode_t *root_inode = NULL;
+        quota_priv_t    *priv = NULL;
 
-unwind:
-	STACK_UNWIND (frame, op_ret, op_errno, statvfs);
-	return 0;
+        priv = this->private;
+
+	if (priv->consider_statfs && loc->inode) {
+		root_inode = loc->inode->table->root;
+		inode_ref(root_inode);
+		STACK_WIND_COOKIE (frame, quota_statfs_cbk, root_inode,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->statfs, loc, xdata);
+	}
+	else {
+		/*
+		 * We have to make sure that we never get to quota_statfs_cbk
+		 * with a cookie that points to something other than an inode,
+		 * which is exactly what would happen with STACK_UNWIND using
+		 * that as a callback.  Therefore, use default_statfs_cbk in
+		 * this case instead.
+                 *
+                 * Also if the option deem-statfs is not set to "on" don't
+                 * bother calculating quota limit on / in statfs_cbk.
+		 */
+                if (priv->consider_statfs)
+                        gf_log(this->name,GF_LOG_WARNING,
+                               "missing inode, cannot adjust for quota");
+		STACK_WIND (frame, default_statfs_cbk, FIRST_CHILD(this),
+			    FIRST_CHILD(this)->fops->statfs, loc, xdata);
+	}
+        return 0;
 }
 
 
 int
-quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+quota_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
 {
-	STACK_WIND (frame, quota_statfs_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->statfs, loc);
+        gf_dirent_t *entry = NULL;
 
-	return 0;
-}
+        if (op_ret <= 0)
+                goto unwind;
 
+        list_for_each_entry (entry, &entries->list, list) {
+                /* TODO: fill things */
+        }
+
+unwind:
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 
+        return 0;
+}
 int
-quota_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, dict_t *value)
-{
-	data_t *data = NULL;
-	struct quota_priv *priv = this->private;
-	
-	if (op_ret >= 0) {
-		data = dict_get (value, "trusted.glusterfs-quota-du");
-		if (data) {
-			LOCK (&priv->lock);
-			{
-				priv->current_disk_usage = data_to_uint64 (data);
-			}
-			UNLOCK (&priv->lock);
+quota_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, dict_t *dict)
+{
+        int ret = 0;
+
+        if (dict) {
+                ret = dict_set_uint64 (dict, QUOTA_SIZE_KEY, 0);
+                if (ret < 0) {
+                        goto err;
+                }
+        }
+
+        STACK_WIND (frame, quota_readdirp_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
+                    fd, size, offset, dict);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (readdirp, frame, -1, EINVAL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+quota_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t                  ret            = 0;
+        uint64_t                 ctx_int        = 0;
+        quota_inode_ctx_t       *ctx            = NULL;
+        quota_local_t           *local          = NULL;
+        quota_dentry_t          *dentry         = NULL;
+        int64_t                  delta          = 0;
+
+        local = frame->local;
+
+        if ((op_ret < 0) || (local == NULL)) {
+                goto out;
+        }
 
-			return 0;
-		}
-	} 
+        ret = inode_ctx_get (local->loc.inode, this, &ctx_int);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get the context", local->loc.path);
+                goto out;
+        }
+
+        ctx = (quota_inode_ctx_t *)(unsigned long) ctx_int;
+
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in %s (gfid:%s)",
+                        local->loc.path, uuid_utoa (local->loc.inode->gfid));
+                goto out;
+        }
+
+        LOCK (&ctx->lock);
+        {
+                ctx->buf = *postbuf;
+        }
+        UNLOCK (&ctx->lock);
+
+        list_for_each_entry (dentry, &ctx->parents, next) {
+                delta = (postbuf->ia_blocks - prebuf->ia_blocks) * 512;
+                quota_update_size (this, local->loc.inode,
+                                   dentry->name, dentry->par, delta);
+        }
 
-	STACK_DESTROY (frame->root);
+out:
+        QUOTA_STACK_UNWIND (fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
 
-	return 0;
+        return 0;
 }
 
+int32_t
+quota_fallocate_helper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		       int32_t mode, off_t offset, size_t len, dict_t *xdata)
+{
+        quota_local_t *local    = NULL;
+        int32_t        op_errno = EINVAL;
 
-void
-gf_quota_get_disk_usage (xlator_t *this)
+        local = frame->local;
+        if (local == NULL) {
+                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
+                goto unwind;
+        }
+
+        if (local->op_ret == -1) {
+                op_errno = local->op_errno;
+                goto unwind;
+        }
+
+        STACK_WIND (frame, quota_fallocate_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		    xdata);
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		off_t offset, size_t len, dict_t *xdata)
+{
+        int32_t            ret     = -1, op_errno = EINVAL;
+        int32_t            parents = 0;
+        quota_local_t     *local   = NULL;
+        quota_inode_ctx_t *ctx     = NULL;
+        quota_priv_t      *priv    = NULL;
+        call_stub_t       *stub    = NULL;
+        quota_dentry_t    *dentry  = NULL;
+
+        GF_ASSERT (frame);
+        GF_VALIDATE_OR_GOTO ("quota", this, unwind);
+        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
+
+        local = quota_local_new ();
+        if (local == NULL) {
+                goto unwind;
+        }
+
+        frame->local = local;
+        local->loc.inode = inode_ref (fd->inode);
+
+        ret = quota_inode_ctx_get (fd->inode, -1, this, NULL, NULL, &ctx, 0);
+        if (ctx == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "quota context not set in inode (gfid:%s)",
+                        uuid_utoa (fd->inode->gfid));
+                goto unwind;
+        }
+
+        stub = fop_fallocate_stub(frame, quota_fallocate_helper, fd, mode, offset, len,
+				  xdata); 
+        if (stub == NULL) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
+
+        priv = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
+
+        LOCK (&ctx->lock);
+        {
+                list_for_each_entry (dentry, &ctx->parents, next) {
+                        parents++;
+                }
+        }
+        UNLOCK (&ctx->lock);
+
+	/*
+	 * Note that by using len as the delta we're assuming the range from
+	 * offset to offset+len has not already been allocated. This can result
+	 * in ENOSPC errors attempting to allocate an already allocated range.
+	 */
+        local->delta = len;
+        local->stub = stub;
+        local->link_count = parents;
+
+        list_for_each_entry (dentry, &ctx->parents, next) {
+                ret = quota_check_limit (frame, fd->inode, this, dentry->name,
+                                         dentry->par);
+                if (ret == -1) {
+                        break;
+                }
+        }
+
+        stub = NULL;
+
+        LOCK (&local->lock);
+        {
+                local->link_count = 0;
+                if (local->validate_count == 0) {
+                        stub = local->stub;
+                        local->stub = NULL;
+                }
+        }
+        UNLOCK (&local->lock);
+
+        if (stub != NULL) {
+                call_resume (stub);
+        }
+
+        return 0;
+
+unwind:
+        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+mem_acct_init (xlator_t *this)
 {
-	call_frame_t *frame = NULL;
-	call_pool_t  *pool = NULL;
-	loc_t         loc;
+        int     ret = -1;
+
+        if (!this)
+                return ret;
 
-	pool = this->ctx->pool;
-	frame = create_frame (this, pool);
-	build_root_loc (this, &loc);
+        ret = xlator_mem_acct_init (this, gf_quota_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
+                        "init failed");
+                return ret;
+        }
 
-	STACK_WIND (frame, quota_getxattr_cbk,
-		    this->children->xlator,
-		    this->children->xlator->fops->getxattr,
-		    &loc,
-		    "trusted.glusterfs-quota-du");
-	return ;
+        return ret;
 }
 
 
-void
-gf_quota_cache_sync (xlator_t *this)
+int32_t
+quota_forget (xlator_t *this, inode_t *inode)
 {
-	struct quota_priv *priv = NULL;
-	call_frame_t      *frame = NULL;
-	dict_t            *dict = get_new_dict ();
-	loc_t              loc;
+        int32_t               ret     = 0;
+        uint64_t              ctx_int = 0;
+        quota_inode_ctx_t    *ctx     = NULL;
+        quota_dentry_t       *dentry  = NULL, *tmp;
 
+        ret = inode_ctx_del (inode, this, &ctx_int);
 
-	priv = this->private;
-	build_root_loc (this, &loc);
+        if (ret < 0) {
+                return 0;
+        }
 
-	frame = create_frame (this, this->ctx->pool);
-	dict_set (dict, "trusted.glusterfs-quota-du", 
-		  data_from_uint64 (priv->current_disk_usage));
+        ctx = (quota_inode_ctx_t *) (long)ctx_int;
 
-	STACK_WIND (frame, quota_setxattr_cbk,
-		    this->children->xlator,
-		    this->children->xlator->fops->setxattr,
-		    &loc, dict, 0);
+        LOCK (&ctx->lock);
+        {
+                list_for_each_entry_safe (dentry, tmp, &ctx->parents, next) {
+                        __quota_dentry_free (dentry);
+                }
+        }
+        UNLOCK (&ctx->lock);
+
+        LOCK_DESTROY (&ctx->lock);
+
+        GF_FREE (ctx);
+
+        return 0;
 }
 
 
 int
-quota_release (xlator_t *this, fd_t *fd)
+quota_parse_limits (quota_priv_t *priv, xlator_t *this, dict_t *xl_options,
+                    struct list_head *old_list)
 {
-	gf_quota_cache_sync (this);
+        int32_t       ret       = -1;
+        char         *str       = NULL;
+        char         *str_val   = NULL;
+        char         *path      = NULL, *saveptr = NULL;
+        uint64_t      value     = 0;
+        limits_t     *quota_lim = NULL, *old = NULL;
+        char         *last_colon= NULL;
+
+        ret = dict_get_str (xl_options, "limit-set", &str);
+
+        if (str) {
+                path = strtok_r (str, ",", &saveptr);
+
+                while (path) {
+                        last_colon = strrchr (path, ':');
+                        *last_colon = '\0';
+                        str_val = last_colon + 1;
+
+                        ret = gf_string2bytesize (str_val, &value);
+                        if (ret != 0)
+                                goto err;
+
+                        QUOTA_ALLOC_OR_GOTO (quota_lim, limits_t, err);
+
+                        quota_lim->path = path;
+
+                        quota_lim->value = value;
+
+                        gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64,
+                                quota_lim->path, quota_lim->value);
+
+                        if (old_list != NULL) {
+                                list_for_each_entry (old, old_list,
+                                                     limit_list) {
+                                        if (strcmp (old->path, quota_lim->path)
+                                            == 0) {
+                                                uuid_copy (quota_lim->gfid,
+                                                           old->gfid);
+                                                break;
+                                        }
+                                }
+                        }
+
+                        LOCK (&priv->lock);
+                        {
+                                list_add_tail (&quota_lim->limit_list,
+                                               &priv->limit_head);
+                        }
+                        UNLOCK (&priv->lock);
+
+                        path = strtok_r (NULL, ",", &saveptr);
+                }
+        } else {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no \"limit-set\" option provided");
+        }
 
-	return 0;
+        LOCK (&priv->lock);
+        {
+                list_for_each_entry (quota_lim, &priv->limit_head, limit_list) {
+                        gf_log (this->name, GF_LOG_INFO, "%s:%"PRId64,
+                                quota_lim->path, quota_lim->value);
+                }
+        }
+        UNLOCK (&priv->lock);
+
+        ret = 0;
+err:
+        return ret;
 }
 
 
-/* notify */
 int32_t
-notify (xlator_t *this,
-	int32_t event,
-	void *data,
-	...)
-{
-	struct quota_priv *priv = this->private;
-	
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-		if (priv->only_first_time) {
-			priv->only_first_time = 0;
-			if (priv->disk_usage_limit) {
-				gf_quota_get_disk_usage (this);
-			}
-		}
-	default:
-		default_notify (this, event, data);
-		break;
-	}
+init (xlator_t *this)
+{
+        int32_t       ret       = -1;
+        quota_priv_t *priv      = NULL;
+
+        if ((this->children == NULL)
+            || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "FATAL: quota (%s) not configured with "
+                        "exactly one child", this->name);
+                return -1;
+        }
+
+        if (this->parents == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile");
+        }
+
+        QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err);
+
+        INIT_LIST_HEAD (&priv->limit_head);
+
+        LOCK_INIT (&priv->lock);
+
+        this->private = priv;
+
+        ret = quota_parse_limits (priv, this, this->options, NULL);
+
+        if (ret) {
+                goto err;
+        }
+
+        GF_OPTION_INIT ("timeout", priv->timeout, int64, err);
+        GF_OPTION_INIT ("deem-statfs", priv->consider_statfs, bool, err);
+
+        this->local_pool = mem_pool_new (quota_local_t, 64);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto err;
+        }
 
-	return 0;
+        ret = 0;
+err:
+        return ret;
 }
 
 
-int32_t 
-init (xlator_t *this)
+void
+__quota_reconfigure_inode_ctx (xlator_t *this, inode_t *inode, limits_t *limit)
 {
-	int     ret  = 0;
-        data_t *data = NULL;
-	struct quota_priv *_private = NULL;
+        int                ret = -1;
+        quota_inode_ctx_t *ctx = NULL;
+
+        GF_VALIDATE_OR_GOTO ("quota", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, limit, out);
+
+        ret = quota_inode_ctx_get (inode, limit->value, this, NULL, NULL, &ctx,
+                                   1);
+        if ((ret == -1) || (ctx == NULL)) {
+                gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
+                        "context in inode(gfid:%s)",
+                        uuid_utoa (inode->gfid));
+                goto out;
+        }
 
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"FATAL: quota should have exactly one child");
-		return -1;
-	}
-	
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
+        LOCK (&ctx->lock);
+        {
+                ctx->limit = limit->value;
+        }
+        UNLOCK (&ctx->lock);
+
+out:
+        return;
+}
 
-	_private = CALLOC (1, sizeof (struct quota_priv));
-        _private->disk_usage_limit = 0;
-        data = dict_get (this->options, "disk-usage-limit");
-        if (data) {
-		if (gf_string2bytesize (data->data, &_private->disk_usage_limit) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR, 
-                                "invalid number '%s' for disk-usage limit", data->data);
-			ret = -1;
-			goto out;
+
+void
+__quota_reconfigure (xlator_t *this, inode_table_t *itable, limits_t *limit)
+{
+        inode_t *inode = NULL;
+
+        if ((this == NULL) || (itable == NULL) || (limit == NULL)) {
+                goto out;
+        }
+
+        if (!uuid_is_null (limit->gfid)) {
+                inode = inode_find (itable, limit->gfid);
+        } else {
+                inode = inode_resolve (itable, limit->path);
+        }
+
+        if (inode != NULL) {
+                __quota_reconfigure_inode_ctx (this, inode, limit);
+        }
+
+out:
+        return;
+}
+
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int32_t           ret   = -1;
+        quota_priv_t     *priv  = NULL;
+        limits_t         *limit = NULL, *next = NULL, *new = NULL;
+        struct list_head  head  = {0, };
+        xlator_t         *top   = NULL;
+        char              found = 0;
+
+        priv = this->private;
+
+        INIT_LIST_HEAD (&head);
+
+        LOCK (&priv->lock);
+        {
+                list_splice_init (&priv->limit_head, &head);
+        }
+        UNLOCK (&priv->lock);
+
+        ret = quota_parse_limits (priv, this, options, &head);
+        if (ret == -1) {
+                gf_log ("quota", GF_LOG_WARNING,
+                        "quota reconfigure failed, "
+                        "new changes will not take effect");
+                goto out;
+        }
+
+        LOCK (&priv->lock);
+        {
+                top = ((glusterfs_ctx_t *)this->ctx)->active->top;
+                GF_ASSERT (top);
+
+                list_for_each_entry (limit, &priv->limit_head, limit_list) {
+                        __quota_reconfigure (this, top->itable, limit);
                 }
 
-		LOCK_INIT (&_private->lock);
-		_private->current_disk_usage = 0;
-	}
-	
-        _private->min_free_disk_limit = 0;
-        data = dict_get (this->options, "min-free-disk-limit");
-        if (data) {
-		if (gf_string2percent (data->data, &_private->min_free_disk_limit) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR, 
-                                "invalid percent '%s' for min-free-disk limit", data->data);
-			ret = -1;
-			goto out;
+                list_for_each_entry_safe (limit, next, &head, limit_list) {
+                        found = 0;
+                        list_for_each_entry (new, &priv->limit_head,
+                                             limit_list) {
+                                if (strcmp (new->path, limit->path) == 0) {
+                                        found = 1;
+                                        break;
+                                }
+                        }
+
+                        if (!found) {
+                                limit->value = -1;
+                                __quota_reconfigure (this, top->itable, limit);
+                        }
+
+                        list_del_init (&limit->limit_list);
+                        GF_FREE (limit);
                 }
-		_private->refresh_interval = 20; /* 20seconds is default */
-		data = dict_get (this->options, "refresh-interval");
-		if (data) {
-			if (gf_string2time (data->data, 
-					    &_private->refresh_interval)!= 0) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"invalid time '%s' for refresh "
-					"interval", data->data);
-				ret = -1;
-				goto out;
-			}
-		}
         }
+        UNLOCK (&priv->lock);
+
+        GF_OPTION_RECONF ("timeout", priv->timeout, options, int64, out);
+        GF_OPTION_RECONF ("deem-statfs", priv->consider_statfs, options, bool,
+                          out);
 
-	_private->only_first_time = 1;
-        this->private = (void *)_private;
-	ret = 0;
- out:
-	return ret;
+        ret = 0;
+out:
+        return ret;
 }
 
-void 
+
+void
 fini (xlator_t *this)
 {
-	struct quota_priv *_private = this->private;
-
-	if (_private) {
-		gf_quota_cache_sync (this);
-		this->private = NULL;
-	}
-	
-	return ;
+        return;
 }
 
-struct xlator_fops fops = {
-	.create      = quota_create,
-	.open        = quota_open,
-	.truncate    = quota_truncate,
-	.ftruncate   = quota_ftruncate,
-	.writev      = quota_writev,
-	.unlink      = quota_unlink,
-	.rmdir       = quota_rmdir,
-	.mknod       = quota_mknod,
-	.mkdir       = quota_mkdir,
-	.symlink     = quota_symlink,
-	.statfs      = quota_statfs,
-};
 
-struct xlator_mops mops = {
+struct xlator_fops fops = {
+        .statfs       = quota_statfs,
+        .lookup       = quota_lookup,
+        .writev       = quota_writev,
+        .create       = quota_create,
+        .mkdir        = quota_mkdir,
+        .truncate     = quota_truncate,
+        .ftruncate    = quota_ftruncate,
+        .unlink       = quota_unlink,
+        .symlink      = quota_symlink,
+        .link         = quota_link,
+        .rename       = quota_rename,
+        .getxattr     = quota_getxattr,
+        .fgetxattr    = quota_fgetxattr,
+        .stat         = quota_stat,
+        .fstat        = quota_fstat,
+        .readlink     = quota_readlink,
+        .readv        = quota_readv,
+        .fsync        = quota_fsync,
+        .setattr      = quota_setattr,
+        .fsetattr     = quota_fsetattr,
+        .mknod        = quota_mknod,
+        .setxattr     = quota_setxattr,
+        .fsetxattr    = quota_fsetxattr,
+        .removexattr  = quota_removexattr,
+        .fremovexattr = quota_fremovexattr,
+        .readdirp     = quota_readdirp,
+	.fallocate    = quota_fallocate,
 };
 
 struct xlator_cbks cbks = {
-	.release     = quota_release
+        .forget = quota_forget
 };
 
 struct volume_options options[] = {
-	{ .key  = {"min-free-disk-limit"}, 
-	  .type = GF_OPTION_TYPE_PERCENT
-	},
-	{ .key  = {"refresh-interval"}, 
-	  .type = GF_OPTION_TYPE_TIME
-	},
-	{ .key  = {"disk-usage-limit"}, 
-	  .type = GF_OPTION_TYPE_SIZET 
-	},
-	{ .key = {NULL} },
+        {.key = {"limit-set"}},
+        {.key = {"timeout"},
+         .type = GF_OPTION_TYPE_SIZET,
+         .min = 0,
+         .max = 60,
+         .default_value = "0",
+         .description = "quota caches the directory sizes on client. Timeout "
+                        "indicates the timeout for the cache to be revalidated."
+        },
+        {.key = {"deem-statfs"},
+         .type = GF_OPTION_TYPE_BOOL,
+         .default_value = "off",
+         .description = "If set to on, it takes quota limits into"
+                        "consideration while estimating fs size. (df command)"
+                        " (Default is off)."
+        },
+        {.key = {NULL}}
 };
diff --git a/xlators/features/quota/src/quota.h b/xlators/features/quota/src/quota.h
new file mode 100644
index 000000000..84ecbb308
--- /dev/null
+++ b/xlators/features/quota/src/quota.h
@@ -0,0 +1,151 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "byte-order.h"
+#include "common-utils.h"
+#include "quota-mem-types.h"
+
+#define QUOTA_XATTR_PREFIX      "trusted."
+#define DIRTY                   "dirty"
+#define SIZE                    "size"
+#define CONTRIBUTION            "contri"
+#define VAL_LENGTH              8
+#define READDIR_BUF             4096
+
+#define QUOTA_SAFE_INCREMENT(lock, var)         \
+        do {                                    \
+                LOCK (lock);                    \
+                var ++;                         \
+                UNLOCK (lock);                  \
+        } while (0)
+
+#define QUOTA_SAFE_DECREMENT(lock, var)         \
+        do {                                    \
+                LOCK (lock);                    \
+                var --;                         \
+                UNLOCK (lock);                  \
+        } while (0)
+
+#define QUOTA_ALLOC_OR_GOTO(var, type, label)           \
+        do {                                            \
+                var = GF_CALLOC (sizeof (type), 1,      \
+                                 gf_quota_mt_##type);   \
+                if (!var) {                             \
+                        gf_log ("", GF_LOG_ERROR,       \
+                                "out of memory :(");    \
+                        ret = -1;                       \
+                        goto label;                     \
+                }                                       \
+        } while (0);
+
+#define QUOTA_STACK_UNWIND(fop, frame, params...)                       \
+        do {                                                            \
+                quota_local_t *_local = NULL;                           \
+                xlator_t      *_this  = NULL;                           \
+                if (frame) {                                            \
+                        _local = frame->local;                          \
+                        _this  = frame->this;                           \
+                        frame->local = NULL;                            \
+                }                                                       \
+                STACK_UNWIND_STRICT (fop, frame, params);               \
+                quota_local_cleanup (_this, _local);                    \
+        } while (0)
+
+#define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)     \
+        do {                                            \
+                list_del (&_contribution->contri_list); \
+                GF_FREE (_contribution);                \
+        } while (0)
+
+#define GET_CONTRI_KEY(var, _vol_name, _gfid, _ret)             \
+        do {                                                    \
+                char _gfid_unparsed[40];                        \
+                uuid_unparse (_gfid, _gfid_unparsed);           \
+                _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
+                                    "%s.%s." CONTRIBUTION,      \
+                                    _vol_name, _gfid_unparsed); \
+        } while (0)
+
+
+#define GET_CONTRI_KEY_OR_GOTO(var, _vol_name, _gfid, label)    \
+        do {                                                    \
+                GET_CONTRI_KEY(var, _vol_name, _gfid, ret);     \
+                if (ret == -1)                                  \
+                        goto label;                             \
+        } while (0)
+
+#define GET_DIRTY_KEY_OR_GOTO(var, _vol_name, label)            \
+        do {                                                    \
+                ret = gf_asprintf (var, QUOTA_XATTR_PREFIX      \
+                                   "%s." DIRTY, _vol_name);     \
+                if (ret == -1)                                  \
+                        goto label;                             \
+        } while (0)
+
+struct quota_dentry {
+        char            *name;
+        uuid_t           par;
+        struct list_head next;
+};
+typedef struct quota_dentry quota_dentry_t;
+
+struct quota_inode_ctx {
+        int64_t          size;
+        int64_t          limit;
+        struct iatt      buf;
+        struct list_head parents;
+        struct timeval   tv;
+        gf_lock_t        lock;
+};
+typedef struct quota_inode_ctx quota_inode_ctx_t;
+
+struct quota_local {
+        gf_lock_t    lock;
+        uint32_t     validate_count;
+        uint32_t     link_count;
+        loc_t        loc;
+        loc_t        oldloc;
+        loc_t        newloc;
+        loc_t        validate_loc;
+        int64_t      delta;
+        int32_t      op_ret;
+        int32_t      op_errno;
+        int64_t      size;
+        int64_t      limit;
+        char         just_validated;
+        inode_t     *inode;
+        call_stub_t *stub;
+};
+typedef struct quota_local quota_local_t;
+
+struct quota_priv {
+        int64_t           timeout;
+        gf_boolean_t      consider_statfs;
+        struct list_head  limit_head;
+        gf_lock_t         lock;
+};
+typedef struct quota_priv quota_priv_t;
+
+struct limits {
+        struct list_head  limit_list;
+        char             *path;
+        int64_t           value;
+        uuid_t            gfid;
+};
+typedef struct limits     limits_t;
+
+uint64_t cn = 1;
diff --git a/xlators/features/read-only/Makefile.am b/xlators/features/read-only/Makefile.am
new file mode 100644
index 000000000..d471a3f92
--- /dev/null
+++ b/xlators/features/read-only/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES = 
diff --git a/xlators/features/read-only/src/Makefile.am b/xlators/features/read-only/src/Makefile.am
new file mode 100644
index 000000000..4c1462137
--- /dev/null
+++ b/xlators/features/read-only/src/Makefile.am
@@ -0,0 +1,22 @@
+xlator_LTLIBRARIES = read-only.la worm.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = read-only-common.h
+
+read_only_la_LDFLAGS = -module -avoid-version
+
+read_only_la_SOURCES = read-only.c read-only-common.c
+read_only_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
+
+worm_la_LDFLAGS = -module -avoid-version
+
+worm_la_SOURCES = read-only-common.c worm.c
+worm_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES = 
+
diff --git a/xlators/features/read-only/src/read-only-common.c b/xlators/features/read-only/src/read-only-common.c
new file mode 100644
index 000000000..56a7a7176
--- /dev/null
+++ b/xlators/features/read-only/src/read-only-common.c
@@ -0,0 +1,239 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+int32_t
+ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (xattrop, frame, -1, EROFS, NULL, xdata);
+        return 0;
+}
+
+int32_t
+ro_fxattrop (call_frame_t *frame, xlator_t *this,
+             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (fxattrop, frame, -1, EROFS, NULL, xdata);
+        return 0;
+}
+
+int32_t
+ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+            loc_t *loc, const char *basename, entrylk_cmd cmd,
+            entrylk_type type, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (entrylk, frame, -1, EROFS, xdata);
+        return 0;
+}
+
+int32_t
+ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (fentrylk, frame, -1, EROFS, xdata);
+        return 0;
+}
+
+int32_t
+ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (inodelk, frame, -1, EROFS, xdata);
+        return 0;
+}
+
+int32_t
+ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (finodelk, frame, -1, EROFS, xdata);
+        return 0;
+}
+
+int32_t
+ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+       struct gf_flock *flock, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (lk, frame, -1, EROFS, NULL, xdata);
+        return 0;
+}
+
+int32_t
+ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (setattr, frame, -1, EROFS, NULL, NULL, xdata);
+	return 0;
+}
+
+int32_t
+ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (fsetattr, frame, -1, EROFS, NULL, NULL, xdata);
+        return 0;
+}
+
+
+int32_t
+ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (truncate, frame, -1, EROFS, NULL, NULL, xdata);
+	return 0;
+}
+
+int32_t
+ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, EROFS, NULL, NULL, xdata);
+	return 0;
+}
+
+int
+ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (mknod, frame, -1, EROFS, NULL, NULL, NULL, NULL, xdata);
+        return 0;
+}
+
+
+int
+ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (mkdir, frame, -1, EROFS, NULL, NULL, NULL, NULL, xdata);
+        return 0;
+}
+
+int32_t
+ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (unlink, frame, -1, EROFS, NULL, NULL, xdata);
+        return 0;
+}
+
+
+int
+ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (rmdir, frame, -1, EROFS, NULL, NULL, xdata);
+        return 0;
+}
+
+
+int
+ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (symlink, frame, -1, EROFS, NULL, NULL, NULL,
+                             NULL, xdata);
+        return 0;
+}
+
+
+
+int32_t
+ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (rename, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                             NULL, xdata);
+        return 0;
+}
+
+
+int32_t
+ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (link, frame, -1, EROFS, NULL, NULL, NULL, NULL, xdata);
+        return 0;
+}
+
+int32_t
+ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (create, frame, -1, EROFS, NULL, NULL, NULL,
+                             NULL, NULL, xdata);
+        return 0;
+}
+
+
+static int32_t
+ro_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+	return 0;
+}
+
+int32_t
+ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
+{
+        if (((flags & O_ACCMODE) == O_WRONLY) ||
+              ((flags & O_ACCMODE) == O_RDWR)) {
+                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, xdata);
+                return 0;
+	}
+
+	STACK_WIND (frame, ro_open_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+	return 0;
+}
+
+int32_t
+ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (fsetxattr, frame, -1, EROFS, xdata);
+	return 0;
+}
+
+int32_t
+ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (fsyncdir, frame, -1, EROFS, xdata);
+	return 0;
+}
+
+int32_t
+ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (writev, frame, -1, EROFS, NULL, NULL, xdata);
+        return 0;
+}
+
+
+int32_t
+ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (setxattr, frame, -1, EROFS, xdata);
+        return 0;
+}
+
+int32_t
+ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (removexattr, frame, -1, EROFS, xdata);
+        return 0;
+}
diff --git a/xlators/features/read-only/src/read-only-common.h b/xlators/features/read-only/src/read-only-common.h
new file mode 100644
index 000000000..5d4c7e260
--- /dev/null
+++ b/xlators/features/read-only/src/read-only-common.h
@@ -0,0 +1,115 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+int32_t
+ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+ro_fxattrop (call_frame_t *frame, xlator_t *this,
+             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+            loc_t *loc, const char *basename, entrylk_cmd cmd,
+            entrylk_type type, dict_t *xdata);
+
+int32_t
+ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type
+             type, dict_t *xdata);
+
+int32_t
+ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+
+int32_t
+ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+
+int32_t
+ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+       struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+int32_t
+ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+
+int32_t
+ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata);
+
+int32_t
+ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata);
+
+int
+ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata);
+
+int
+ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
+
+int32_t
+ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
+
+int
+ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
+
+
+int
+ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata);
+
+int32_t
+ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+
+int32_t
+ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+
+int32_t
+ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+
+int32_t
+ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata);
+
+int32_t
+ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
+
+int32_t
+ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata);
+
+int32_t
+ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata);
+
+int32_t
+ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
+
+int32_t
+ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
diff --git a/xlators/features/read-only/src/read-only.c b/xlators/features/read-only/src/read-only.c
new file mode 100644
index 000000000..e49e54a1b
--- /dev/null
+++ b/xlators/features/read-only/src/read-only.c
@@ -0,0 +1,77 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "read-only-common.h"
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+
+struct xlator_fops fops = {
+        .mknod       = ro_mknod,
+        .mkdir       = ro_mkdir,
+        .unlink      = ro_unlink,
+        .rmdir       = ro_rmdir,
+        .symlink     = ro_symlink,
+        .rename      = ro_rename,
+        .link        = ro_link,
+        .truncate    = ro_truncate,
+        .open        = ro_open,
+        .writev      = ro_writev,
+        .setxattr    = ro_setxattr,
+        .fsetxattr   = ro_fsetxattr,
+        .removexattr = ro_removexattr,
+        .fsyncdir    = ro_fsyncdir,
+        .ftruncate   = ro_ftruncate,
+        .create      = ro_create,
+        .setattr     = ro_setattr,
+        .fsetattr    = ro_fsetattr,
+        .xattrop     = ro_xattrop,
+        .fxattrop    = ro_fxattrop,
+        .inodelk     = ro_inodelk,
+        .finodelk    = ro_finodelk,
+        .entrylk     = ro_entrylk,
+        .fentrylk    = ro_fentrylk,
+        .lk          = ro_lk,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/features/read-only/src/worm.c b/xlators/features/read-only/src/worm.c
new file mode 100644
index 000000000..16c3eb3da
--- /dev/null
+++ b/xlators/features/read-only/src/worm.c
@@ -0,0 +1,89 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "read-only-common.h"
+
+static int32_t
+worm_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+               int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int32_t
+worm_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
+{
+        if ((((flags & O_ACCMODE) == O_WRONLY) ||
+              ((flags & O_ACCMODE) == O_RDWR)) &&
+              !(flags & O_APPEND)) {
+                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, NULL);
+                return 0;
+        }
+
+        STACK_WIND (frame, worm_open_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+        return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "translator not configured with exactly one child");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+        return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        return;
+}
+
+struct xlator_fops fops = {
+        .open        = worm_open,
+
+        .unlink      = ro_unlink,
+        .rmdir       = ro_rmdir,
+        .rename      = ro_rename,
+        .truncate    = ro_truncate,
+        .removexattr = ro_removexattr,
+        .fsyncdir    = ro_fsyncdir,
+        .xattrop     = ro_xattrop,
+        .inodelk     = ro_inodelk,
+        .finodelk    = ro_finodelk,
+        .entrylk     = ro_entrylk,
+        .fentrylk    = ro_fentrylk,
+        .lk          = ro_lk,
+};
+
+struct xlator_cbks cbks;
+
+struct volume_options options[] = {
+        { .key = {NULL} },
+};
+
diff --git a/xlators/features/trash/src/Makefile.am b/xlators/features/trash/src/Makefile.am
index d61f608aa..5251eb082 100644
--- a/xlators/features/trash/src/Makefile.am
+++ b/xlators/features/trash/src/Makefile.am
@@ -1,13 +1,16 @@
 xlator_LTLIBRARIES = trash.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-trash_la_LDFLAGS = -module -avoidversion 
+trash_la_LDFLAGS = -module -avoid-version 
 
 trash_la_SOURCES = trash.c
 trash_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = trash.h trash-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/trash/src/trash-mem-types.h b/xlators/features/trash/src/trash-mem-types.h
new file mode 100644
index 000000000..0e6ef572f
--- /dev/null
+++ b/xlators/features/trash/src/trash-mem-types.h
@@ -0,0 +1,22 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __TRASH_MEM_TYPES_H__
+#define __TRASH_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_trash_mem_types_ {
+        gf_trash_mt_trash_private_t = gf_common_mt_end + 1,
+        gf_trash_mt_char,
+        gf_trash_mt_trash_elim_pattern_t,
+        gf_trash_mt_end
+};
+#endif
+
diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c
index c8e7357ee..addeb66a0 100644
--- a/xlators/features/trash/src/trash.c
+++ b/xlators/features/trash/src/trash.c
@@ -1,512 +1,1392 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
+#include "trash.h"
+#include "trash-mem-types.h"
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+int32_t
+trash_ftruncate_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iovec *vector, int32_t count,
+                           struct iatt *stbuf, struct iobref *iobuf);
 
-#include <libgen.h>
+int32_t
+trash_truncate_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iatt *prebuf, struct iatt *postbuf);
 
-/* TODO: currently it can work only above posix, no other translators 
- *       between them. Not a good thing. Try making more reliable methods.
- */
+int32_t
+trash_truncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *stbuf, struct iatt *preparent,
+                          struct iatt *postparent);
 
-struct trash_struct {
-	inode_t *inode;
-	loc_t loc1;
-	loc_t loc2;
-	char origpath[ZR_PATH_MAX];
-	char newpath[ZR_PATH_MAX];
-	char oldpath[ZR_PATH_MAX]; // used only in case of rename
-};
-typedef struct trash_struct trash_local_t;
+int32_t
+trash_unlink_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         struct iatt *preoldparent, struct iatt *postoldparent,
+                         struct iatt *prenewparent, struct iatt *postnewparent);
 
-struct trash_priv {
-	char trash_dir[ZR_PATH_MAX];
-};
-typedef struct trash_priv trash_private_t;
+void
+trash_local_wipe (trash_local_t *local)
+{
+        if (!local)
+                goto out;
+
+        loc_wipe (&local->loc);
+        loc_wipe (&local->newloc);
+
+        if (local->fd)
+                fd_unref (local->fd);
+
+        if (local->newfd)
+                fd_unref (local->newfd);
+
+        mem_put (local);
+out:
+        return;
+}
+
+int32_t
+trash_common_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno,
+                         struct iatt *preparent, struct iatt *postparent)
+{
+        TRASH_STACK_UNWIND (unlink, frame, op_ret, op_errno, preparent, postparent);
+        return 0;
+}
 
 int32_t
-trash_unlink_rename_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 struct stat *buf);
+trash_unlink_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent)
+{
+        trash_local_t *local       = NULL;
+        char          *tmp_str     = NULL;
+        char          *tmp_path    = NULL;
+        char          *tmp_dirname = NULL;
+        char          *dir_name    = NULL;
+        int32_t        count       = 0;
+        int32_t        loop_count  = 0;
+        int            i           = 0;
+        loc_t          tmp_loc     = {0,};
+
+        local   = frame->local;
+        tmp_str = gf_strdup (local->newpath);
+        if (!tmp_str) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto out;
+        }
+        loop_count = local->loop_count;
+
+        if ((op_ret == -1) &&  (op_errno == ENOENT)) {
+                tmp_dirname = strchr (tmp_str, '/');
+                while (tmp_dirname) {
+                        count = tmp_dirname - tmp_str;
+                        if (count == 0)
+                                count = 1;
+                        i++;
+                        if (i > loop_count)
+                                break;
+                        tmp_dirname = strchr (tmp_str + count + 1, '/');
+                }
+                tmp_path = memdup (local->newpath, count);
+                if (!tmp_path) {
+                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                        goto out;
+                }
+
+                tmp_loc.path = tmp_path;
+
+                /* TODO:create the directory with proper permissions */
+                STACK_WIND_COOKIE (frame, trash_unlink_mkdir_cbk, tmp_path,
+                                   this->children->xlator,
+                                   this->children->xlator->fops->mkdir,
+                                   &tmp_loc, 0755, NULL);
+
+                goto out;
+        }
+
+        if (op_ret == 0) {
+                dir_name = dirname (tmp_str);
+                if (strcmp((char*)cookie, dir_name) == 0) {
+                        tmp_loc.path = local->newpath;
+                        STACK_WIND (frame, trash_unlink_rename_cbk,
+                                    this->children->xlator,
+                                    this->children->xlator->fops->rename,
+                                    &local->loc, &tmp_loc);
+                        goto out;
+                }
+        }
+
+        LOCK (&frame->lock);
+        {
+                loop_count = ++local->loop_count;
+        }
+        UNLOCK (&frame->lock);
+        tmp_dirname = strchr (tmp_str, '/');
+        while (tmp_dirname) {
+                count = tmp_dirname - tmp_str;
+                if (count == 0)
+                        count = 1;
+                i++;
+                if ((i > loop_count) || (count > PATH_MAX))
+                        break;
+                tmp_dirname = strchr (tmp_str + count + 1, '/');
+        }
+        tmp_path = memdup (local->newpath, count);
+        if (!tmp_path) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                goto out;
+        }
+        tmp_loc.path = tmp_path;
+
+        STACK_WIND_COOKIE (frame, trash_unlink_mkdir_cbk, tmp_path,
+                           this->children->xlator,
+                           this->children->xlator->fops->mkdir,
+                           &tmp_loc, 0755, NULL);
+
+out:
+        GF_FREE (cookie);
+        GF_FREE (tmp_str);
+
+        return 0;
+}
+
 int32_t
-trash_rename_rename_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 struct stat *buf);
+trash_rename_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent);
 
-/**
- * trash_common_unwind_cbk -
- */
 int32_t
-trash_common_unwind_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
+trash_unlink_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         struct iatt *preoldparent, struct iatt *postoldparent,
+                         struct iatt *prenewparent, struct iatt *postnewparent)
 {
-	trash_local_t *local = frame->local;
+        trash_local_t   *local      = NULL;
+        char            *tmp_str    = NULL;
+        char            *dir_name   = NULL;
+        char            *tmp_cookie = NULL;
+        loc_t            tmp_loc    = {0,};
+
+        local = frame->local;
+
+        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                tmp_str = gf_strdup (local->newpath);
+                if (!tmp_str) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                dir_name = dirname (tmp_str);
+
+                tmp_loc.path = dir_name;
+
+                tmp_cookie = gf_strdup (dir_name);
+                if (!tmp_cookie) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                /* TODO: create the directory with proper permissions */
+                STACK_WIND_COOKIE (frame, trash_unlink_mkdir_cbk, tmp_cookie,
+                                   FIRST_CHILD(this),
+                                   FIRST_CHILD(this)->fops->mkdir,
+                                   &tmp_loc, 0755, NULL);
+
+                GF_FREE (tmp_str);
 
-	if (local->loc1.path)
-		loc_wipe (&local->loc1);
-	
-	if (local->loc2.path)
-		loc_wipe (&local->loc2);
+                return 0;
+        }
 
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
+        if ((op_ret == -1) && (op_errno == ENOTDIR)) {
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "target(%s) exists, cannot keep the copy, deleting",
+                        local->newpath);
+
+                STACK_WIND (frame, trash_common_unwind_cbk,
+                            this->children->xlator,
+                            this->children->xlator->fops->unlink, &local->loc);
+
+                return 0;
+        }
+
+        if ((op_ret == -1) && (op_errno == EISDIR)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "target(%s) exists as directory, cannot keep copy, "
+                        "deleting", local->newpath);
+
+                STACK_WIND (frame, trash_common_unwind_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->unlink, &local->loc);
+                return 0;
+        }
+
+        /* All other cases, unlink should return success */
+        TRASH_STACK_UNWIND (unlink, frame, 0, op_errno, &local->preparent,
+                            &local->postparent);
+
+        return 0;
 }
 
-/**
- * trash_common_unwind_buf_cbk -
- */
+
+
+int32_t
+trash_common_unwind_buf_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *prebuf, struct iatt *postbuf)
+{
+        TRASH_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf, postbuf);
+        return 0;
+}
+
+int
+trash_common_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                         struct iatt *preoldparent, struct iatt *postoldparent,
+                         struct iatt *prenewparent, struct iatt *postnewparent)
+{
+        TRASH_STACK_UNWIND (rename, frame, op_ret, op_errno, stbuf, preoldparent,
+                            postoldparent, prenewparent, postnewparent);
+        return 0;
+}
+
+
 int32_t
-trash_common_unwind_buf_cbk (call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct stat *buf)
+trash_unlink_stat_cbk (call_frame_t *frame,  void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *buf)
 {
-	trash_local_t *local = frame->local;
+        trash_private_t *priv    = NULL;
+        trash_local_t   *local   = NULL;
+        loc_t            new_loc = {0,};
+
+        priv  = this->private;
+        local = frame->local;
 
-	if (local->loc1.path)
-		loc_wipe (&local->loc1);
-	
-	if (local->loc2.path)
-		loc_wipe (&local->loc2);
+        if (-1 == op_ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s: %s",
+                        local->loc.path, strerror (op_errno));
+                goto fail;
+        }
+
+        if ((buf->ia_size == 0) ||
+            (buf->ia_size > priv->max_trash_file_size)) {
+                /* if the file is too big or zero, just unlink it */
+
+                if (buf->ia_size > priv->max_trash_file_size) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s: file size too big (%"PRId64") to "
+                                "move into trash directory",
+                                local->loc.path, buf->ia_size);
+                }
+
+                STACK_WIND (frame, trash_common_unwind_cbk,
+                            this->children->xlator,
+                            this->children->xlator->fops->unlink, &local->loc);
+                return 0;
+        }
+
+        new_loc.path = local->newpath;
+
+        STACK_WIND (frame, trash_unlink_rename_cbk,
+                    this->children->xlator,
+                    this->children->xlator->fops->rename,
+                    &local->loc, &new_loc);
+
+        return 0;
+
+fail:
+        TRASH_STACK_UNWIND (unlink, frame, op_ret, op_errno, buf,
+                            NULL);
+
+        return 0;
 
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
 }
 
 int32_t
-trash_mkdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-		 struct stat *stbuf)
+trash_rename_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         struct iatt *preoldparent, struct iatt *postoldparent,
+                         struct iatt *prenewparent, struct iatt *postnewparent)
 {
-	trash_local_t *local = frame->local;
-	char *tmp_str = strdup (local->newpath);
-	int32_t count = 0;
-	char *tmp_path = NULL;
-	char *tmp_dirname = NULL;
-
-	if (op_ret == -1 && op_errno == ENOENT) {
-		tmp_dirname = strchr (tmp_str, '/');
-		while (tmp_dirname) {
-			count = tmp_dirname - tmp_str;
-			if (count == 0)
-				count = 1;
-			tmp_path = CALLOC (1, count + 1);
-			ERR_ABORT (tmp_path);
-			memcpy (tmp_path, local->newpath, count);
-			loc_t tmp_loc = {
-				.inode = NULL,
-				.path = tmp_path,
-			};
-
-			/* TODO:create the directory with proper permissions */
-			STACK_WIND_COOKIE (frame,
-					   trash_mkdir_cbk,
-					   tmp_path,
-					   this->children->xlator,
-					   this->children->xlator->fops->mkdir,
-					   &tmp_loc,
-					   0777);
-			tmp_dirname = strchr (tmp_str + count + 1, '/');
-		}
-		free (cookie);
-		free (tmp_str);
-		return 0;
-	}
-	char *dir_name = dirname (tmp_str);
-	if (strcmp((char*)cookie, dir_name) == 0) {
-		loc_t new_loc = {
-			.inode = NULL,
-			.path = local->newpath
-		};
-		STACK_WIND (frame,
-			    trash_unlink_rename_cbk,
-			    this->children->xlator,
-			    this->children->xlator->fops->rename,
-			    &local->loc2,
-			    &new_loc);
-
-	}
-	free (cookie); /* strdup (dir_name) was sent here :) */
-	free (tmp_str);
-	return 0;
+        trash_local_t *local    = NULL;
+        char          *tmp_str  = NULL;
+        char          *dir_name = NULL;
+        char          *tmp_path = NULL;
+        loc_t          tmp_loc  = {0,};
+
+        local = frame->local;
+        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                tmp_str  = gf_strdup (local->newpath);
+                if (!tmp_str) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                dir_name = dirname (tmp_str);
+
+                /* check for the errno, if its ENOENT create directory and call
+                 * rename later
+                 */
+                tmp_path = gf_strdup (dir_name);
+                if (!tmp_path) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                tmp_loc.path = tmp_path;
+
+                /* TODO: create the directory with proper permissions */
+                STACK_WIND_COOKIE (frame, trash_rename_mkdir_cbk, tmp_path,
+                                   this->children->xlator,
+                                   this->children->xlator->fops->mkdir,
+                                   &tmp_loc, 0755, NULL);
+
+                GF_FREE (tmp_str);
+                return 0;
+        }
+
+        if ((op_ret == -1) && (op_errno == ENOTDIR)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "target(%s) exists, cannot keep the dest entry(%s): "
+                        "renaming", local->newpath, local->origpath);
+        } else if ((op_ret == -1) && (op_errno == EISDIR)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "target(%s) exists as a directory, cannot keep the "
+                        "copy (%s), renaming", local->newpath, local->origpath);
+        }
+
+        STACK_WIND (frame, trash_common_rename_cbk,
+                    this->children->xlator,
+                    this->children->xlator->fops->rename, &local->loc,
+                    &local->newloc);
+
+        return 0;
+}
+
+
+int32_t
+trash_rename_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent)
+{
+        trash_local_t *local = NULL;
+        char          *tmp_str = NULL;
+        char          *tmp_path = NULL;
+        char          *tmp_dirname = NULL;
+        char          *dir_name = NULL;
+        int32_t        count = 0;
+        loc_t          tmp_loc = {0,};
+
+        local   = frame->local;
+        tmp_str = gf_strdup (local->newpath);
+        if (!tmp_str) {
+                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                goto out;
+        }
+
+        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                tmp_dirname = strchr (tmp_str, '/');
+                while (tmp_dirname) {
+                        count = tmp_dirname - tmp_str;
+                        if (count == 0)
+                                count = 1;
+
+                        tmp_dirname = strchr (tmp_str + count + 1, '/');
+
+                        tmp_path = memdup (local->newpath, count);
+                        if (!tmp_path) {
+                                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                        }
+
+                        tmp_loc.path = tmp_path;
+
+                        /* TODO: create the directory with proper permissions */
+                        STACK_WIND_COOKIE (frame, trash_rename_mkdir_cbk,
+                                           tmp_path,  this->children->xlator,
+                                           this->children->xlator->fops->mkdir,
+                                           &tmp_loc, 0755, NULL);
+                }
+
+                goto out;
+        }
+
+        dir_name = dirname (tmp_str);
+        if (strcmp ((char*)cookie, dir_name) == 0) {
+                tmp_loc.path = local->newpath;
+
+                STACK_WIND (frame, trash_rename_rename_cbk,
+                            this->children->xlator,
+                            this->children->xlator->fops->rename,
+                            &local->newloc, &tmp_loc);
+        }
+
+out:
+        GF_FREE (cookie); /* strdup (dir_name) was sent here :) */
+        GF_FREE (tmp_str);
+
+        return 0;
 }
 
-/**
- * trash_unlink_rename_cbk -
- */
 int32_t
-trash_unlink_rename_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 struct stat *buf)
+trash_rename_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, dict_t *xattr,
+                         struct iatt *postparent)
 {
-	trash_local_t *local = frame->local;
-	if (op_ret == -1 && op_errno == ENOENT) {
-		/* check for the errno, if its ENOENT create directory and call
-		 * rename later
-		 */
-		char *tmp_str = strdup (local->newpath);
-		char *dir_name = dirname (tmp_str);
-		loc_t tmp_loc = {
-			.inode = NULL,
-			.path = dir_name,
-		};
-		/* TODO: create the directory with proper permissions */
-		STACK_WIND_COOKIE (frame,
-				   trash_mkdir_cbk,
-				   strdup (dir_name),
-				   this->children->xlator,
-				   this->children->xlator->fops->mkdir,
-				   &tmp_loc,
-				   0777);
-		free (tmp_str);
-	} else if (op_ret == -1 && op_errno == ENOTDIR) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Target exists, cannot keep the copy, deleting");
-		STACK_WIND (frame,
-			    trash_common_unwind_cbk,
-			    this->children->xlator,
-			    this->children->xlator->fops->unlink,
-			    &local->loc2);
-	} else if (op_ret == -1 && op_errno == EISDIR) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Target exists as a directory, cannot keep the copy, "
-			"deleting");
-		STACK_WIND (frame,
-			    trash_common_unwind_cbk,
-			    this->children->xlator,
-			    this->children->xlator->fops->unlink,
-			    &local->loc2);
-	} else {
-		/* */
-		STACK_UNWIND (frame, 0, op_errno);
-	}
-
-	return 0;
+        trash_private_t *priv = NULL;
+        trash_local_t   *local = NULL;
+        loc_t            tmp_loc = {0,};
+
+        local = frame->local;
+        priv  = this->private;
+
+        if (op_ret == -1) {
+                STACK_WIND (frame, trash_common_rename_cbk,
+                            this->children->xlator,
+                            this->children->xlator->fops->rename,
+                            &local->loc, &local->newloc);
+                return 0;
+        }
+        if ((buf->ia_size == 0) ||
+            (buf->ia_size > priv->max_trash_file_size)) {
+                /* if the file is too big or zero, just unlink it */
+
+                if (buf->ia_size > priv->max_trash_file_size) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s: file size too big (%"PRId64") to "
+                                "move into trash directory",
+                                local->newloc.path, buf->ia_size);
+                }
+
+                STACK_WIND (frame, trash_common_rename_cbk,
+                            this->children->xlator,
+                            this->children->xlator->fops->rename,
+                            &local->loc, &local->newloc);
+                return 0;
+        }
+
+        tmp_loc.path = local->newpath;
+
+        STACK_WIND (frame, trash_rename_rename_cbk,
+                    this->children->xlator,
+                    this->children->xlator->fops->rename,
+                    &local->newloc, &tmp_loc);
+
+        return 0;
 }
 
 
-/**
- * trash_unlink -
- */
 int32_t
-trash_unlink (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
+trash_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+              loc_t *newloc)
 {
-	trash_private_t *priv = this->private;
-	trash_local_t *local = NULL;
-	time_t       utime = 0;
-	struct tm   *tm = NULL;
-	char         timestr[256];
-
-	if (strncmp (loc->path, priv->trash_dir, 
-		     strlen(priv->trash_dir)) == 0) {
-		/* Trying to rename from the trash can dir, do the
-		   actual unlink */
-		STACK_WIND (frame,
-			    trash_common_unwind_cbk,
-			    this->children->xlator,
-			    this->children->xlator->fops->unlink,
-			    loc);
-	} else {
-		local = CALLOC (1, sizeof (trash_local_t));
-		if (!local) {
-			STACK_UNWIND (frame, -1, ENOMEM);
-			return 0;
-		}
-		frame->local = local;
-		
-		loc_copy (&local->loc2, loc);
-
-		strcpy (local->newpath, priv->trash_dir);
-		strcat (local->newpath, loc->path);
-
-		utime = time (NULL);
-		tm    = localtime (&utime);
-		strftime (timestr, 256, ".%Y%m%d%H%M%S", tm); 
-		strcat (local->newpath, timestr);
-
-		{
-			loc_t new_loc = {
-				.inode = NULL,
-				.path = local->newpath
-			};
-			STACK_WIND (frame,
-				    trash_unlink_rename_cbk,
-				    this->children->xlator,
-				    this->children->xlator->fops->rename,
-				    loc,
-				    &new_loc);
-		}
-	}
-	return 0;
+        trash_elim_pattern_t  *trav  = NULL;
+        trash_private_t *priv = NULL;
+        trash_local_t   *local = NULL;
+        char             timestr[64] = {0,};
+        int32_t          match = 0;
+
+        priv = this->private;
+        if (priv->eliminate) {
+                trav = priv->eliminate;
+                while (trav) {
+                        if (fnmatch(trav->pattern, newloc->name, 0) == 0) {
+                                match++;
+                                break;
+                        }
+                        trav = trav->next;
+                }
+        }
+
+        if ((strncmp (oldloc->path, priv->trash_dir,
+                      strlen (priv->trash_dir)) == 0) || match) {
+                /* Trying to rename from the trash dir,
+                   do the actual rename */
+                STACK_WIND (frame, trash_common_rename_cbk,
+                            this->children->xlator,
+                            this->children->xlator->fops->rename,
+                            oldloc, newloc);
+
+                return 0;
+        }
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                TRASH_STACK_UNWIND (rename, frame, -1, ENOMEM,
+                                    NULL, NULL, NULL, NULL, NULL);
+                return 0;
+        }
+
+        frame->local = local;
+        loc_copy (&local->loc, oldloc);
+
+        loc_copy (&local->newloc, newloc);
+
+        strcpy (local->origpath, newloc->path);
+        strcpy (local->newpath, priv->trash_dir);
+        strcat (local->newpath, newloc->path);
+
+        {
+                /* append timestamp to file name */
+                /* TODO: can we make it optional? */
+                gf_time_ftm (timestr, sizeof timestr, time (NULL),
+                             gf_timefmt_F_HMS);
+                strcat (local->newpath, timestr);
+        }
+
+        /* Send a lookup call on newloc, to ensure we are not
+           overwriting */
+        STACK_WIND (frame, trash_rename_lookup_cbk,
+                    this->children->xlator,
+                    this->children->xlator->fops->lookup, newloc, 0);
+
+        return 0;
 }
 
-/* */
 int32_t
-trash_rename_mkdir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			inode_t *inode,
-			struct stat *stbuf)
+trash_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-	trash_local_t *local = frame->local;
-	char *tmp_str = strdup (local->newpath);
-
-	if (op_ret == -1 && op_errno == ENOENT) {
-		int32_t count = 0;
-		char *tmp_path = NULL;
-		char *tmp_dirname = strchr (tmp_str, '/');
-
-		while (tmp_dirname) {
-			count = tmp_dirname - tmp_str;
-			if (count == 0)
-				count = 1;
-			tmp_path = CALLOC (1, count + 2);
-			ERR_ABORT (tmp_path);
-			memcpy (tmp_path, local->newpath, count);
-			loc_t tmp_loc = {
-				.inode = NULL,
-				.path = tmp_path,
-			};
-
-			/* TODO:create the directory with proper permissions */
-			STACK_WIND_COOKIE (frame,
-					   trash_rename_mkdir_cbk,
-					   tmp_path,
-					   this->children->xlator,
-					   this->children->xlator->fops->mkdir,
-					   &tmp_loc,
-					   0777);
-			tmp_dirname = strchr (tmp_str + count + 1, '/');
-		}
-		free (cookie);
-		free (tmp_str);
-		return 0;
-	}
-	char *dir_name = dirname (tmp_str);
-	if (strcmp((char*)cookie, dir_name) == 0) {
-		loc_t new_loc = {
-			.inode = NULL,
-			.path = local->newpath
-		};
-		STACK_WIND (frame,
-			    trash_rename_rename_cbk,
-			    this->children->xlator,
-			    this->children->xlator->fops->rename,
-			    &local->loc2,
-			    &new_loc);
-
-	}
-	free (cookie); /* strdup (dir_name) was sent here :) */
-	free (tmp_str);
-	return 0;
+        trash_elim_pattern_t  *trav  = NULL;
+        trash_private_t *priv = NULL;
+        trash_local_t   *local = NULL;
+        char             timestr[64] = {0,};
+        int32_t          match = 0;
+
+        priv = this->private;
+
+        if (priv->eliminate) {
+                trav = priv->eliminate;
+                while (trav) {
+                        if (fnmatch(trav->pattern, loc->name, 0) == 0) {
+                                match++;
+                                break;
+                        }
+                        trav = trav->next;
+                }
+        }
+
+        if ((strncmp (loc->path, priv->trash_dir,
+                      strlen (priv->trash_dir)) == 0) || (match)) {
+                if (match) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s: file matches eliminate pattern, "
+                                "not moved to trash", loc->name);
+                } else {
+                        /* unlink from the trash-dir, not keeping any copy */
+                        ;
+                }
+
+                STACK_WIND (frame, trash_common_unwind_cbk,
+                            this->children->xlator,
+                            this->children->xlator->fops->unlink, loc);
+                return 0;
+        }
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                TRASH_STACK_UNWIND (unlink, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+        frame->local = local;
+        loc_copy (&local->loc, loc);
+
+        strcpy (local->origpath, loc->path);
+        strcpy (local->newpath, priv->trash_dir);
+        strcat (local->newpath, loc->path);
+
+        {
+                /* append timestamp to file name */
+                /* TODO: can we make it optional? */
+                gf_time_fmt (timestr, sizeof timestr, time (NULL),
+                             gf_timefmt_F_HMS);
+                strcat (local->newpath, timestr);
+        }
+
+        LOCK_INIT (&frame->lock);
+
+        STACK_WIND (frame, trash_unlink_stat_cbk,
+                    this->children->xlator,
+                    this->children->xlator->fops->stat, loc);
+
+        return 0;
 }
 
+int32_t
+trash_truncate_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iatt *preparent, struct iatt *postparent)
+{
+        /* use this Function when a failure occurs, and
+           delete the newly created file. */
+        trash_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "deleting the newly created file: %s",
+                        strerror (op_errno));
+        }
+
+        STACK_WIND (frame, trash_common_unwind_buf_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+                    &local->loc, local->fop_offset);
+
+        return 0;
+}
 
-/**
- * trash_unlink_rename_cbk -
- */
 int32_t
-trash_rename_rename_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 struct stat *buf)
+trash_truncate_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno,
+                          struct iovec *vector, int32_t count,
+                          struct iatt *stbuf, struct iobref *iobuf)
 {
-	trash_local_t *local = frame->local;
-	if (op_ret == -1 && op_errno == ENOENT) {
-		/* check for the errno, if its ENOENT create directory and call
-		 * rename later
-		 */
-		char *tmp_str = strdup (local->newpath);
-		char *dir_name = dirname (tmp_str);
-		loc_t tmp_loc = {
-			.inode = NULL,
-			.path = dir_name,
-		};
-		/* TODO: create the directory with proper permissions */
-		STACK_WIND_COOKIE (frame,
-				   trash_rename_mkdir_cbk,
-				   strdup (dir_name),
-				   this->children->xlator,
-				   this->children->xlator->fops->mkdir,
-				   &tmp_loc,
-				   0777);
-		free (tmp_str);
-		return 0;
-	} else if (op_ret == -1 && op_errno == ENOTDIR) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Target exists, cannot keep the dest entry %s, "
-			"renaming",
-			local->loc2.path);
-	} else if (op_ret == -1 && op_errno == EISDIR) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Target exists as a directory, cannot keep the "
-			"copy %s, renaming",
-			local->loc2.path);
-	}
-	loc_t new_loc = {
-		.inode = NULL,
-		.parent = local->loc2.parent,
-		.path = local->loc2.path,
-	};
-	STACK_WIND (frame,
-		    trash_common_unwind_buf_cbk,
-		    this->children->xlator,
-		    this->children->xlator->fops->rename,
-		    &local->loc1,
-		    &new_loc);
-
-	return 0;
+        trash_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "readv on the existing file failed: %s",
+                        strerror (op_errno));
+
+                STACK_WIND (frame, trash_truncate_unlink_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                            &local->newloc);
+                goto out;
+        }
+
+        local->fsize = stbuf->ia_size;
+        STACK_WIND (frame, trash_truncate_writev_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev,
+                    local->newfd, vector, count, local->cur_offset, 0, iobuf);
+
+out:
+        return 0;
+
 }
 
-/**
- * trash_rename_lookup_cbk -
- */
 int32_t
-trash_rename_lookup_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 inode_t *inode,
-			 struct stat *buf,
-			 dict_t *xattr)
+trash_truncate_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iatt *prebuf, struct iatt *postbuf)
 {
-	trash_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		STACK_WIND (frame,
-			    trash_common_unwind_buf_cbk,
-			    this->children->xlator,
-			    this->children->xlator->fops->rename,
-			    &local->loc1,
-			    &local->loc2);
-		return 0;
-	}
-
-	loc_t oldloc = {
-		.parent = local->loc2.parent,
-		.inode = inode,
-		.path = local->loc2.path,
-	};
-	loc_t newloc = {
-		.inode = NULL,
-		.path = local->newpath
-	};
-	STACK_WIND (frame,
-		    trash_rename_rename_cbk,
-		    this->children->xlator,
-		    this->children->xlator->fops->rename,
-		    &oldloc,
-		    &newloc);
-
-	return 0;
+        trash_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                /* Let truncate work, but previous copy is not preserved. */
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "writev on the existing file failed: %s",
+                        strerror (op_errno));
+
+                STACK_WIND (frame, trash_truncate_unlink_cbk, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->unlink, &local->newloc);
+                goto out;
+        }
+
+        if (local->cur_offset < local->fsize) {
+                local->cur_offset += GF_BLOCK_READV_SIZE;
+                /* Loop back and Read the contents again. */
+                STACK_WIND (frame, trash_truncate_readv_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                            local->fd, (size_t)GF_BLOCK_READV_SIZE,
+                            local->cur_offset, 0);
+                goto out;
+        }
+
+
+        /* OOFH.....Finally calling Truncate. */
+        STACK_WIND (frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->truncate, &local->loc,
+                    local->fop_offset);
+
+out:
+        return 0;
+}
+
+
+
+int32_t
+trash_truncate_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, fd_t *fd)
+{
+        trash_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                //Let truncate work, but previous copy is not preserved.
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "open on the existing file failed: %s",
+                        strerror (op_errno));
+
+                STACK_WIND (frame, trash_truncate_unlink_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                            &local->newloc);
+                goto out;
+        }
+
+        local->cur_offset = local->fop_offset;
+
+        STACK_WIND (frame, trash_truncate_readv_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
+                    local->fd, (size_t)GF_BLOCK_READV_SIZE, local->cur_offset, 0);
+
+out:
+        return 0;
+}
+
+
+int32_t
+trash_truncate_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, fd_t *fd,
+                           inode_t *inode, struct iatt *buf,
+                           struct iatt *preparent, struct iatt *postparent)
+{
+        trash_local_t       *local    = NULL;
+        char                *tmp_str  = NULL;
+        char                *dir_name = NULL;
+        char                *tmp_path = NULL;
+        int32_t              flags    = 0;
+        loc_t                tmp_loc  = {0,};
+
+        local = frame->local;
+
+        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                //Creating the directory structure here.
+                tmp_str = gf_strdup (local->newpath);
+                if (!tmp_str) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                dir_name = dirname (tmp_str);
+
+                tmp_path = gf_strdup (dir_name);
+                if (!tmp_path) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                tmp_loc.path = tmp_path;
+
+                /* TODO: create the directory with proper permissions */
+                STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk,
+                                   tmp_path, FIRST_CHILD(this),
+                                   FIRST_CHILD(this)->fops->mkdir,
+                                   &tmp_loc, 0755, NULL);
+                GF_FREE (tmp_str);
+                goto out;
+        }
+
+        if (op_ret == -1) {
+                //Let truncate work, but previous copy is not preserved.
+                //Deleting the newly created copy.
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "creation of new file in trash-dir failed, "
+                        "when truncate was called: %s", strerror (op_errno));
+
+                STACK_WIND (frame, trash_common_unwind_buf_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->truncate, &local->loc,
+                            local->fop_offset);
+                goto out;
+        }
+
+        flags = O_RDONLY;
+
+        local->fd = fd_create (local->loc.inode, frame->root->pid);
+
+        STACK_WIND (frame, trash_truncate_open_cbk,  FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open, &local->loc, flags,
+                    local->fd, 0);
+out:
+        return 0;
+}
+
+int32_t
+trash_truncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *stbuf, struct iatt *preparent,
+                          struct iatt *postparent)
+{
+        trash_local_t       *local = NULL;
+        char                *tmp_str = NULL;
+        char                *tmp_path = NULL;
+        char                *tmp_dirname = NULL;
+        char                *dir_name = NULL;
+        int32_t              count = 0;
+        int32_t              flags = 0;
+        int32_t              loop_count = 0;
+        int                  i = 0;
+        loc_t                tmp_loc = {0,};
+
+        local   = frame->local;
+        if (!local)
+                goto out;
+
+        loop_count = local->loop_count;
+
+        tmp_str = gf_strdup (local->newpath);
+        if (!tmp_str) {
+                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                goto out;
+        }
+
+        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                tmp_dirname = strchr (tmp_str, '/');
+                while (tmp_dirname) {
+                        count = tmp_dirname - tmp_str;
+                        if (count == 0)
+                                count = 1;
+                        i++;
+                        if (i > loop_count)
+                                break;
+                        tmp_dirname = strchr (tmp_str + count + 1, '/');
+                }
+                tmp_path = memdup (local->newpath, count);
+                if (!tmp_path) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                tmp_loc.path = tmp_path;
+                STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk,
+                                   tmp_path, this->children->xlator,
+                                   this->children->xlator->fops->mkdir,
+                                   &tmp_loc, 0755, NULL);
+
+                goto out;
+        }
+
+        if (op_ret == 0) {
+                dir_name = dirname (tmp_str);
+                if (strcmp ((char*)cookie, dir_name) == 0) {
+                        flags = O_CREAT|O_EXCL|O_WRONLY;
+                        ia_prot_t prot = {0, };
+
+                        //Call create again once directory structure is created.
+                        STACK_WIND (frame, trash_truncate_create_cbk,
+                                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                                    &local->newloc, flags,
+                                    st_mode_from_ia (prot, local->loc.inode->ia_type),
+                                    local->newfd, NULL);
+                        goto out;
+                }
+        }
+
+        LOCK (&frame->lock);
+        {
+                loop_count = ++local->loop_count;
+        }
+        UNLOCK (&frame->lock);
+
+        tmp_dirname = strchr (tmp_str, '/');
+        while (tmp_dirname) {
+                count = tmp_dirname - tmp_str;
+                if (count == 0)
+                        count = 1;
+
+                i++;
+                if ((i > loop_count) || (count > PATH_MAX))
+                        break;
+                tmp_dirname = strchr (tmp_str + count + 1, '/');
+        }
+        tmp_path = memdup (local->newpath, count);
+        if (!tmp_path) {
+                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+        }
+        tmp_loc.path = tmp_path;
+
+        STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk, tmp_path,
+                           this->children->xlator,
+                           this->children->xlator->fops->mkdir,
+                           &tmp_loc, 0755, NULL);
+
+out:
+        GF_FREE (cookie); /* strdup (dir_name) was sent here :) */
+        GF_FREE (tmp_str);
+
+        return 0;
+}
+
+
+int32_t
+trash_truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf)
+{
+        trash_private_t     *priv  = NULL;
+        trash_local_t       *local = NULL;
+        char                 timestr[64] = {0,};
+        char                 loc_newname[PATH_MAX] = {0,};
+        int32_t              flags = 0;
+
+        priv = this->private;
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "fstat on the file failed: %s",
+                        strerror (op_errno));
+
+                TRASH_STACK_UNWIND (truncate, frame, op_ret, op_errno, buf, NULL);
+                return 0;
+        }
+
+        if ((buf->ia_size == 0) || (buf->ia_size > priv->max_trash_file_size)) {
+                // If the file is too big, just unlink it.
+                if (buf->ia_size > priv->max_trash_file_size)
+                        gf_log (this->name, GF_LOG_DEBUG, "%s: file too big, "
+                                "not moving to trash", local->loc.path);
+
+                STACK_WIND (frame,  trash_common_unwind_buf_cbk,
+                            this->children->xlator,
+                            this->children->xlator->fops->truncate,
+                            &local->loc, local->fop_offset);
+                return 0;
+        }
+
+        strcpy (local->newpath, priv->trash_dir);
+        strcat (local->newpath, local->loc.path);
+
+        {
+                gf_time_fmt (timestr, sizeof timestr, time (NULL),
+                             gf_timefmt_F_HMS);
+                strcat (local->newpath, timestr);
+        }
+        strcpy (loc_newname,local->loc.name);
+        strcat (loc_newname,timestr);
+
+        local->newloc.name = gf_strdup (loc_newname);
+        local->newloc.path = gf_strdup (local->newpath);
+        local->newloc.inode = inode_new (local->loc.inode->table);
+        local->newfd = fd_create (local->newloc.inode, frame->root->pid);
+
+        flags = O_CREAT|O_EXCL|O_WRONLY;
+
+        STACK_WIND (frame, trash_truncate_create_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create,
+                    &local->newloc, flags,
+                    st_mode_from_ia (buf->ia_prot, local->loc.inode->ia_type),
+                    local->newfd, NULL);
+
+        return 0;
+}
+
+int32_t
+trash_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                off_t offset)
+{
+        trash_elim_pattern_t *trav = NULL;
+        trash_private_t      *priv = NULL;
+        trash_local_t        *local = NULL;
+        int32_t               match = 0;
+
+        priv  = this->private;
+        if (priv->eliminate) {
+                trav = priv->eliminate;
+                while (trav) {
+                        if (fnmatch(trav->pattern, loc->name, 0) == 0) {
+                                match++;
+                                break;
+                        }
+                        trav = trav->next;
+                }
+        }
+
+        if ((strncmp (loc->path, priv->trash_dir,
+                      strlen (priv->trash_dir)) == 0) || (offset) || (match)) {
+                if (match) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "%s: file not moved to trash as per option "
+                                "'eliminate'", loc->path);
+                }
+
+                // Trying to truncate from the trash can dir,
+                //   do the actual truncate without moving to trash-dir.
+                STACK_WIND (frame, trash_common_unwind_buf_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->truncate, loc, offset);
+                goto out;
+        }
+
+        LOCK_INIT (&frame->lock);
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                TRASH_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        loc_copy (&local->loc, loc);
+
+        local->fop_offset = offset;
+
+        frame->local = local;
+
+        STACK_WIND (frame, trash_truncate_stat_cbk,
+                    this->children->xlator,
+                    this->children->xlator->fops->stat, loc);
+
+out:
+        return 0;
+}
+
+int32_t
+trash_ftruncate_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *preparent, struct iatt *postparent)
+{
+        trash_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: failed to unlink new file: %s",
+                        local->newloc.path, strerror(op_errno));
+
+        }
+
+        STACK_WIND (frame, trash_common_unwind_buf_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate,
+                    local->fd, local->fop_offset);
+
+        return 0;
+}
+
+int32_t
+trash_ftruncate_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *prebuf, struct iatt *postbuf)
+{
+        trash_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                STACK_WIND (frame, trash_ftruncate_unlink_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                            &local->newloc);
+                return 0;
+        }
+
+        if (local->cur_offset < local->fsize) {
+                local->cur_offset += GF_BLOCK_READV_SIZE;
+                STACK_WIND (frame, trash_ftruncate_readv_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                            local->fd, (size_t)GF_BLOCK_READV_SIZE,
+                            local->cur_offset, 0);
+                return 0;
+        }
+
+        STACK_WIND (frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, local->fd,
+                    local->fop_offset);
+
+        return 0;
+}
+
+
+int32_t
+trash_ftruncate_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iovec *vector, int32_t count,
+                           struct iatt *stbuf, struct iobref *iobuf)
+{
+        trash_local_t *local = NULL;
+
+        local = frame->local;
+        local->fsize = stbuf->ia_size;
+
+        if (op_ret == -1) {
+                STACK_WIND (frame, trash_ftruncate_unlink_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                            &local->newloc);
+                return 0;
+        }
+
+        STACK_WIND (frame, trash_ftruncate_writev_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+                    local->newfd, vector, count, local->cur_offset, 0, NULL);
+
+        return 0;
+}
+
+
+int32_t
+trash_ftruncate_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, fd_t *fd,
+                            inode_t *inode, struct iatt *buf,
+                            struct iatt *preparent, struct iatt *postparent)
+{
+        trash_local_t *local = NULL;
+        char          *tmp_str = NULL;
+        char          *dir_name = NULL;
+        char          *tmp_path = NULL;
+        loc_t          tmp_loc = {0,};
+
+        local = frame->local;
+
+        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                tmp_str = gf_strdup (local->newpath);
+                if (!tmp_str) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                dir_name = dirname (tmp_str);
+
+                tmp_path = gf_strdup (dir_name);
+                if (!tmp_path) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                tmp_loc.path = tmp_path;
+
+                /* TODO: create the directory with proper permissions */
+                STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk,
+                                   tmp_path, FIRST_CHILD(this),
+                                   FIRST_CHILD(this)->fops->mkdir,
+                                   &tmp_loc, 0755, NULL);
+                GF_FREE (tmp_str);
+                return 0;
+        }
+
+        if (op_ret == -1) {
+                STACK_WIND (frame, trash_common_unwind_buf_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->ftruncate,
+                            local->fd, local->fop_offset);
+                return 0;
+        }
+
+        STACK_WIND (frame, trash_ftruncate_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, local->fd,
+                    (size_t)GF_BLOCK_READV_SIZE, local->cur_offset, 0);
+
+        return 0;
 }
 
 
-/**
- * trash_rename -
- */
 int32_t
-trash_rename (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *oldloc,
-	      loc_t *newloc)
+trash_ftruncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *stbuf, struct iatt *preparent,
+                           struct iatt *postparent)
 {
-	trash_private_t *priv = this->private;
-	trash_local_t *local = NULL;
-	time_t       utime = 0;
-	struct tm   *tm = NULL;
-	char         timestr[256];
-
-	if (strncmp (oldloc->path, priv->trash_dir, 
-		     strlen(priv->trash_dir)) == 0) {
-		/* Trying to rename from the trash can dir, 
-		   do the actual rename */
-		STACK_WIND (frame,
-			    trash_common_unwind_buf_cbk,
-			    this->children->xlator,
-			    this->children->xlator->fops->rename,
-			    oldloc,
-			    newloc);
-	} else {
-		/* Trying to rename a regular file from GlusterFS */
-		local = CALLOC (1, sizeof (trash_local_t));
-		if (!local) {
-			STACK_UNWIND (frame, -1, ENOMEM, NULL);
-			return 0;
-		}
-		frame->local = local;
-		loc_copy (&local->loc1, oldloc);
-		loc_copy (&local->loc2, newloc);
-
-		strcpy (local->newpath, priv->trash_dir);
-		strcat (local->newpath, newloc->path);
-
-		utime = time (NULL);
-		tm    = localtime (&utime);
-		strftime (timestr, 256, ".%Y%m%d%H%M%S", tm); 
-		strcat (local->newpath, timestr);
-
-		/* Send a lookup call on newloc, to ensure we are not 
-		   overwriting */
-		STACK_WIND (frame,
-			    trash_rename_lookup_cbk,
-			    this->children->xlator,
-			    this->children->xlator->fops->lookup,
-			    newloc,
-			    0);
-	}
-	return 0;
+        trash_local_t       *local = NULL;
+        char                *tmp_str = NULL;
+        char                *tmp_path = NULL;
+        char                *tmp_dirname = NULL;
+        char                *dir_name = NULL;
+        int32_t              count = 0;
+        int32_t              flags = 0;
+        int32_t              loop_count = 0;
+        int                  i = 0;
+        loc_t                tmp_loc = {0,};
+
+        local   = frame->local;
+        if (!local)
+                goto out;
+
+        loop_count = local->loop_count;
+
+        tmp_str = gf_strdup (local->newpath);
+        if (!tmp_str) {
+                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                goto out;
+        }
+
+        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                tmp_dirname = strchr (tmp_str, '/');
+                while (tmp_dirname) {
+                        count = tmp_dirname - tmp_str;
+                        if (count == 0)
+                                count = 1;
+                        i++;
+                        if (i > loop_count)
+                                break;
+                        tmp_dirname = strchr (tmp_str + count + 1, '/');
+                }
+                tmp_path = memdup (local->newpath, count);
+                if (!tmp_path) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+                tmp_loc.path = tmp_path;
+                STACK_WIND_COOKIE (frame, trash_ftruncate_mkdir_cbk,
+                                   tmp_path, this->children->xlator,
+                                   this->children->xlator->fops->mkdir,
+                                   &tmp_loc, 0755, NULL);
+
+                goto out;
+        }
+
+        if (op_ret == 0) {
+                dir_name = dirname (tmp_str);
+                if (strcmp ((char*)cookie, dir_name) == 0) {
+                        ia_prot_t prot = {0, };
+                        flags = O_CREAT|O_EXCL|O_WRONLY;
+
+                        //Call create again once directory structure is created.
+                        STACK_WIND (frame, trash_ftruncate_create_cbk,
+                                    FIRST_CHILD(this),
+                                    FIRST_CHILD(this)->fops->create,
+                                    &local->newloc, flags,
+                                    st_mode_from_ia (prot, local->loc.inode->ia_type),
+                                    local->newfd, NULL);
+                        goto out;
+                }
+        }
+
+        LOCK (&frame->lock);
+        {
+                loop_count = ++local->loop_count;
+        }
+        UNLOCK (&frame->lock);
+        tmp_dirname = strchr (tmp_str, '/');
+        while (tmp_dirname) {
+                count = tmp_dirname - tmp_str;
+                if (count == 0)
+                        count = 1;
+
+                i++;
+                if ((i > loop_count) || (count > PATH_MAX))
+                        break;
+                tmp_dirname = strchr (tmp_str + count + 1, '/');
+        }
+        tmp_path = memdup (local->newpath, count);
+        if (!tmp_path) {
+                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+        }
+        tmp_loc.path = tmp_path;
+
+        STACK_WIND_COOKIE (frame, trash_ftruncate_mkdir_cbk, tmp_path,
+                           this->children->xlator,
+                           this->children->xlator->fops->mkdir,
+                           &tmp_loc, 0755, NULL);
+
+out:
+        GF_FREE (cookie); /* strdup (dir_name) was sent here :) */
+        GF_FREE (tmp_str);
+
+        return 0;
+}
+
+
+int32_t
+trash_ftruncate_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, struct iatt *buf)
+{
+        trash_private_t *priv  = NULL;
+        trash_local_t   *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: %s",local->newloc.path, strerror(op_errno));
+
+                TRASH_STACK_UNWIND (ftruncate, frame, -1, op_errno, buf, NULL);
+                return 0;
+        }
+        if ((buf->ia_size == 0) || (buf->ia_size > priv->max_trash_file_size))
+        {
+                STACK_WIND (frame, trash_common_unwind_buf_cbk,
+                            this->children->xlator,
+                            this->children->xlator->fops->ftruncate,
+                            local->fd, local->fop_offset);
+                return 0;
+        }
+
+
+        STACK_WIND (frame, trash_ftruncate_create_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create, &local->newloc,
+                    ( O_CREAT | O_EXCL | O_WRONLY ),
+                    st_mode_from_ia (buf->ia_prot, local->loc.inode->ia_type),
+                    local->newfd, NULL);
+
+        return 0;
+}
+
+int32_t
+trash_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+{
+        trash_elim_pattern_t  *trav = NULL;
+        trash_private_t       *priv = NULL;
+        trash_local_t         *local = NULL;
+        dentry_t              *dir_entry = NULL;
+        char                  *pathbuf = NULL;
+        inode_t               *newinode = NULL;
+        char                   timestr[64];
+        int32_t                retval = 0;
+        int32_t                match = 0;
+
+        priv = this->private;
+
+        dir_entry = __dentry_search_arbit (fd->inode);
+        retval = inode_path (fd->inode, NULL, &pathbuf);
+
+        if (priv->eliminate) {
+                trav = priv->eliminate;
+                while (trav) {
+                        if (fnmatch(trav->pattern, dir_entry->name, 0) == 0) {
+                                match++;
+                                break;
+                        }
+                        trav = trav->next;
+                }
+        }
+
+        if ((strncmp (pathbuf, priv->trash_dir,
+                      strlen (priv->trash_dir)) == 0) ||
+            (offset >= priv->max_trash_file_size) ||
+            (!retval) ||
+            match) {
+                STACK_WIND (frame, trash_common_unwind_buf_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->ftruncate,
+                            fd, offset);
+                return 0;
+        }
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                TRASH_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        gf_time_fmt (timestr, sizeof timestr, time (NULL), gf_timefmt_F_HMS);
+        strcpy (local->newpath, priv->trash_dir);
+        strcat (local->newpath, pathbuf);
+        strcat (local->newpath, timestr);
+
+        local->fd = fd_ref (fd);
+        newinode = inode_new (fd->inode->table);
+        local->newfd = fd_create (newinode, frame->root->pid);
+        frame->local=local;
+
+        local->newloc.inode = newinode;
+        local->newloc.path = local->newpath;
+
+        local->loc.inode = inode_ref (fd->inode);
+        local->loc.path  = pathbuf;
+
+        local->fop_offset = offset;
+        local->cur_offset = offset;
+
+        STACK_WIND (frame, trash_ftruncate_fstat_cbk, this->children->xlator,
+                    this->children->xlator->fops->fstat, fd);
+
+        return 0;
 }
 
 /**
@@ -515,82 +1395,143 @@ trash_rename (call_frame_t *frame,
 int32_t
 init (xlator_t *this)
 {
-  	data_t *trash_dir = NULL;
-	xlator_list_t *trav = NULL;
-	trash_private_t *_priv = NULL;
-
-	/* Create .trashcan directory in init */
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"not configured with exactly one child. exiting");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	trav = this->children;
-	while (trav->xlator->children)
-		trav = trav->xlator->children;
-
-	if (strncmp ("storage/", trav->xlator->type, 8))
-	{
-		gf_log (this->name, GF_LOG_ERROR,
-			"'trash' translator not loaded over storage "
-			"translator, not a supported setup");
-		return -1;
-	}
-
-	_priv = CALLOC (1, sizeof (*_priv));
-	ERR_ABORT (_priv);
-
-	trash_dir = dict_get (this->options, "trash-dir");
-	if (!trash_dir) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"no option specified for 'trash-dir', "
-			"using \"/.trashcan/\"");
-		strcpy (_priv->trash_dir, "/.trashcan");
-	} else {
-		/* Need a path with '/' as the first char, if not 
-		   given, append it */
-		if (trash_dir->data[0] == '/') {
-			strcpy (_priv->trash_dir, trash_dir->data);
-		} else {
-			strcpy (_priv->trash_dir, "/");
-			strcat (_priv->trash_dir, trash_dir->data);
-		}
-	}
-
-	this->private = (void *)_priv;
-	return 0;
+        data_t                *data  = NULL;
+        trash_private_t       *_priv = NULL;
+        trash_elim_pattern_t  *trav  = NULL;
+        char                  *tmp_str = NULL;
+        char                  *strtokptr = NULL;
+        char                  *component = NULL;
+        char                   trash_dir[PATH_MAX] = {0,};
+        uint64_t               max_trash_file_size64 = 0;
+
+        /* Create .trashcan directory in init */
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "not configured with exactly one child. exiting");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+        }
+
+        _priv = GF_CALLOC (1, sizeof (*_priv), gf_trash_mt_trash_private_t);
+        if (!_priv) {
+                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+                return -1;
+        }
+
+        data = dict_get (this->options, "trash-dir");
+        if (!data) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no option specified for 'trash-dir', "
+                        "using \"/.trashcan/\"");
+                _priv->trash_dir = gf_strdup ("/.trashcan");
+        } else {
+                /* Need a path with '/' as the first char, if not
+                   given, append it */
+                if (data->data[0] == '/') {
+                        _priv->trash_dir = gf_strdup (data->data);
+                } else {
+                        /* TODO: Make sure there is no ".." in the path */
+                        strcpy (trash_dir, "/");
+                        strcat (trash_dir, data->data);
+                        _priv->trash_dir = gf_strdup (trash_dir);
+                }
+        }
+
+        data = dict_get (this->options, "eliminate-pattern");
+        if (!data) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "no option specified for 'eliminate', using NULL");
+        } else {
+                tmp_str = gf_strdup (data->data);
+                if (!tmp_str) {
+                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                }
+
+                /*  Match Filename to option specified in eliminate. */
+                component = strtok_r (tmp_str, "|", &strtokptr);
+                while (component) {
+                        trav = GF_CALLOC (1, sizeof (*trav),
+                                          gf_trash_mt_trash_elim_pattern_t);
+                        if (!trav) {
+                                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
+                                break;
+                        }
+                        trav->pattern = component;
+                        trav->next = _priv->eliminate;
+                        _priv->eliminate = trav;
+
+                        component = strtok_r (NULL, "|", &strtokptr);
+                }
+        }
+
+        /* TODO: do gf_string2sizet () */
+        data = dict_get (this->options, "max-trashable-file-size");
+        if (!data) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no option specified for 'max-trashable-file-size', "
+                        "using default = %lld MB",
+                        GF_DEFAULT_MAX_FILE_SIZE / GF_UNIT_MB);
+                _priv->max_trash_file_size = GF_DEFAULT_MAX_FILE_SIZE;
+        } else {
+                (void)gf_string2bytesize (data->data,
+                                          &max_trash_file_size64);
+                if( max_trash_file_size64 > GF_ALLOWED_MAX_FILE_SIZE ) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Size specified for max-size(in MB) is too "
+                                "large so using 1GB as max-size (NOT IDEAL)");
+                        _priv->max_trash_file_size = GF_ALLOWED_MAX_FILE_SIZE;
+                } else
+                        _priv->max_trash_file_size = max_trash_file_size64;
+                gf_log (this->name, GF_LOG_DEBUG, "%"GF_PRI_SIZET" max-size",
+                        _priv->max_trash_file_size);
+        }
+
+        this->local_pool = mem_pool_new (trash_local_t, 64);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                return -1;
+        }
+
+
+        this->private = (void *)_priv;
+        return 0;
 }
 
 void
 fini (xlator_t *this)
 {
-	trash_private_t *priv = this->private;
-	FREE (priv);
-	return;
-}
+        trash_private_t *priv = NULL;
 
+        priv = this->private;
+        GF_FREE (priv);
 
-struct xlator_fops fops = {
-	.unlink = trash_unlink,
-	.rename = trash_rename,
-};
-
-struct xlator_mops mops = {
+        return;
+}
 
+struct xlator_fops fops = {
+        .unlink    = trash_unlink,
+        .rename    = trash_rename,
+        .truncate  = trash_truncate,
+        .ftruncate = trash_ftruncate,
 };
 
 struct xlator_cbks cbks = {
 };
 
 struct volume_options options[] = {
-	{ .key  = { "trash-dir" }, 
-	  .type = GF_OPTION_TYPE_PATH 
-	},
-	{ .key  = {NULL} },
+        { .key  = { "trash-directory" },
+          .type = GF_OPTION_TYPE_PATH,
+        },
+        { .key  = { "eliminate-pattern" },
+          .type = GF_OPTION_TYPE_STR,
+        },
+        { .key  = { "max-trashable-file-size" },
+          .type = GF_OPTION_TYPE_SIZET,
+        },
+        { .key  = {NULL} },
 };
diff --git a/xlators/features/trash/src/trash.h b/xlators/features/trash/src/trash.h
new file mode 100644
index 000000000..9a7c03361
--- /dev/null
+++ b/xlators/features/trash/src/trash.h
@@ -0,0 +1,79 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __TRASH_H__
+#define __TRASH_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "defaults.h"
+#include "inode.c"
+#include "fnmatch.h"
+
+#include <libgen.h>
+
+#ifndef GF_BLOCK_READV_SIZE
+#define GF_BLOCK_READV_SIZE      (128 * GF_UNIT_KB)
+#endif
+
+#ifndef GF_DEFAULT_MAX_FILE_SIZE
+#define GF_DEFAULT_MAX_FILE_SIZE (200 * GF_UNIT_MB)
+#endif
+
+#ifndef GF_ALLOWED_MAX_FILE_SIZE
+#define GF_ALLOWED_MAX_FILE_SIZE (1 * GF_UNIT_GB)
+#endif
+
+
+struct trash_struct {
+        fd_t    *fd;         /* for the fd of existing file */
+        fd_t    *newfd;      /* for the newly created file */
+        loc_t    loc;        /* to store the location of the existing file */
+        loc_t    newloc;     /* to store the location for the new file */
+        size_t   fsize;      /* for keeping the size of existing file */
+        off_t    cur_offset; /* current offset for read and write ops */
+        off_t    fop_offset;
+        char     origpath[PATH_MAX];
+        char     newpath[PATH_MAX];
+        int32_t  loop_count;
+        struct iatt preparent;
+        struct iatt postparent;
+};
+typedef struct trash_struct trash_local_t;
+
+struct _trash_elim_pattern;
+typedef struct _trash_elim_pattern {
+        struct _trash_elim_pattern *next;
+        char                       *pattern;
+} trash_elim_pattern_t;
+
+struct trash_priv {
+        char                 *trash_dir;
+        trash_elim_pattern_t *eliminate;
+        size_t                max_trash_file_size;
+};
+typedef struct trash_priv trash_private_t;
+
+#define TRASH_STACK_UNWIND(op, frame, params ...) do {     \
+		trash_local_t *__local = NULL;         \
+		__local = frame->local;                \
+		frame->local = NULL;		       \
+		STACK_UNWIND_STRICT (op, frame, params);          \
+		trash_local_wipe (__local);	       \
+	} while (0)
+
+
+#endif /* __TRASH_H__ */
diff --git a/xlators/lib/src/libxlator.c b/xlators/lib/src/libxlator.c
new file mode 100644
index 000000000..9e5357255
--- /dev/null
+++ b/xlators/lib/src/libxlator.c
@@ -0,0 +1,470 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include "mem-types.h"
+#include "libxlator.h"
+
+
+int marker_xtime_default_gauge[] = {
+        [MCNT_FOUND]    =  1,
+        [MCNT_NOTFOUND] = -1,
+        [MCNT_ENODATA]  = -1,
+        [MCNT_ENOTCONN] = -1,
+        [MCNT_ENOENT]   = -1,
+        [MCNT_EOTHER]   = -1,
+};
+
+int marker_uuid_default_gauge[] = {
+        [MCNT_FOUND]    =  1,
+        [MCNT_NOTFOUND] =  0,
+        [MCNT_ENODATA]  =  0,
+        [MCNT_ENOTCONN] =  0,
+        [MCNT_ENOENT]   =  0,
+        [MCNT_EOTHER]   =  0,
+};
+
+static int marker_idx_errno_map[] = {
+        [MCNT_FOUND]    = EINVAL,
+        [MCNT_NOTFOUND] = EINVAL,
+        [MCNT_ENOENT]   = ENOENT,
+        [MCNT_ENOTCONN] = ENOTCONN,
+        [MCNT_ENODATA]  = ENODATA,
+        [MCNT_EOTHER]   = EINVAL,
+        [MCNT_MAX]      = 0,
+};
+
+/*Copy the contents of oldtimebuf to newtimbuf*/
+static void
+update_timebuf (uint32_t *oldtimbuf, uint32_t *newtimebuf)
+{
+        newtimebuf[0] =  (oldtimbuf[0]);
+        newtimebuf[1] =  (oldtimbuf[1]);
+}
+
+/* Convert Timebuf in network order to host order */
+static void
+get_hosttime (uint32_t *oldtimbuf, uint32_t *newtimebuf)
+{
+        newtimebuf[0] = ntohl (oldtimbuf[0]);
+        newtimebuf[1] = ntohl (oldtimbuf[1]);
+}
+
+
+
+/* Match the Incoming trusted.glusterfs.<uuid>.xtime against volume uuid */
+int
+match_uuid_local (const char *name, char *uuid)
+{
+        name = strtail ((char *)name, MARKER_XATTR_PREFIX);
+        if (!name || name++[0] != '.')
+                return -1;
+
+        name = strtail ((char *)name, uuid);
+        if (!name || strcmp (name, ".xtime") != 0)
+                return -1;
+
+        return 0;
+}
+
+static void
+marker_local_incr_errcount (xl_marker_local_t *local, int op_errno)
+{
+        marker_result_idx_t i = -1;
+
+        if (!local)
+                return;
+
+        switch (op_errno) {
+                case ENODATA:
+                        i = MCNT_ENODATA;
+                        break;
+                case ENOENT:
+                        i = MCNT_ENOENT;
+                        break;
+                case ENOTCONN:
+                        i = MCNT_ENOTCONN;
+                        break;
+                default:
+                        i = MCNT_EOTHER;
+                        break;
+        }
+
+        local->count[i]++;
+}
+
+static int
+evaluate_marker_results (int *gauge, int *count)
+{
+        int i = 0;
+        int op_errno = 0;
+        gf_boolean_t sane = _gf_true;
+
+        /* check if the policy of the gauge is violated;
+         * if yes, try to get the best errno, ie. look
+         * for the first position where there is a more
+         * specific kind of vioilation than the generic EINVAL
+         */
+        for (i = 0; i < MCNT_MAX; i++) {
+                if (sane) {
+                        if ((gauge[i] > 0 && count[i] < gauge[i]) ||
+                            (gauge[i] < 0 && count[i] >= -gauge[i])) {
+                                sane = _gf_false;
+                                /* generic action: adopt corresponding errno */
+                                op_errno = marker_idx_errno_map[i];
+                        }
+                } else {
+                        /* already insane; trying to get a more informative
+                         * errno by checking subsequent counters
+                         */
+                        if (count[i] > 0)
+                                op_errno = marker_idx_errno_map[i];
+                }
+                if (op_errno && op_errno != EINVAL)
+                        break;
+        }
+
+        return op_errno;
+}
+
+/* Aggregate all the <volid>.xtime attrs of the cluster and send the max*/
+int32_t
+cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+
+{
+
+        int32_t           callcnt         = 0;
+        int               ret             = -1;
+        uint32_t          *net_timebuf    = NULL;
+        uint32_t          host_timebuf[2] = {0,};
+        char              *marker_xattr   = NULL;
+        xl_marker_local_t *local          = NULL;
+        char              *vol_uuid       = NULL;
+        char              need_unwind     = 0;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("", GF_LOG_DEBUG, "possible NULL deref");
+                need_unwind = 1;
+                goto out;
+        }
+
+        local = frame->local;
+        if (!local || !local->vol_uuid) {
+                gf_log (this->name, GF_LOG_DEBUG, "possible NULL deref");
+                need_unwind = 1;
+                goto out;
+        }
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                vol_uuid = local->vol_uuid;
+
+                if (op_ret) {
+                        marker_local_incr_errcount (local, op_errno);
+                        goto unlock;
+                }
+
+                if (!gf_asprintf (&marker_xattr, "%s.%s.%s",
+                                MARKER_XATTR_PREFIX, vol_uuid, XTIME)) {
+                        op_errno = ENOMEM;
+                        goto unlock;
+                }
+
+
+                if (dict_get_ptr (dict, marker_xattr, (void **)&net_timebuf)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Unable to get <uuid>.xtime attr");
+                        local->count[MCNT_NOTFOUND]++;
+                        goto unlock;
+                }
+
+                if (local->count[MCNT_FOUND]) {
+                        get_hosttime (net_timebuf, host_timebuf);
+                        if ( (host_timebuf[0]>local->host_timebuf[0]) ||
+                                (host_timebuf[0] == local->host_timebuf[0] &&
+                                 host_timebuf[1] >= local->host_timebuf[1])) {
+                                update_timebuf (net_timebuf, local->net_timebuf);
+                                update_timebuf (host_timebuf, local->host_timebuf);
+                        }
+
+                } else {
+                        get_hosttime (net_timebuf, local->host_timebuf);
+                        update_timebuf (net_timebuf, local->net_timebuf);
+                        local->count[MCNT_FOUND]++;
+                }
+
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                op_ret = 0;
+                op_errno = 0;
+                need_unwind = 1;
+
+                if (local->count[MCNT_FOUND]) {
+                        if (!dict)
+                                dict = dict_new();
+
+                        ret = dict_set_static_bin (dict, marker_xattr,
+                                           (void *)local->net_timebuf, 8);
+                        if (ret) {
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+                                goto out;
+                        }
+                }
+
+                op_errno = evaluate_marker_results (local->gauge, local->count);
+                if (op_errno)
+                        op_ret = -1;
+        }
+
+out:
+        if (need_unwind && local && local->xl_specf_unwind) {
+                frame->local = local->xl_local;
+                local->xl_specf_unwind (frame, op_ret,
+                                        op_errno, dict, xdata);
+        } else if (need_unwind) {
+                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno,
+                                     dict, xdata);
+        }
+
+        GF_FREE (marker_xattr);
+        return 0;
+
+}
+
+int32_t
+cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+{
+        int32_t             callcnt     = 0;
+        struct volume_mark  *volmark    = NULL;
+        xl_marker_local_t   *local      = NULL;
+        int32_t             ret         = -1;
+        char                need_unwind = 0;
+        char                *vol_uuid   = NULL;
+
+        if (!this || !frame || !cookie) {
+                gf_log ("", GF_LOG_DEBUG, "possible NULL deref");
+                need_unwind = 1;
+                goto out;
+        }
+
+        local = frame->local;
+
+        if (!local) {
+                gf_log (this->name, GF_LOG_DEBUG, "possible NULL deref");
+                need_unwind = 1;
+                goto out;
+        }
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                vol_uuid = local->vol_uuid;
+
+                if (op_ret) {
+                        marker_local_incr_errcount (local, op_errno);
+                        goto unlock;
+                }
+
+                ret = dict_get_bin (dict, GF_XATTR_MARKER_KEY,
+                                    (void *)&volmark);
+                if (ret)
+                        goto unlock;
+
+                if (local->count[MCNT_FOUND]) {
+                        if ((local->volmark->major != volmark->major) ||
+                            (local->volmark->minor != volmark->minor)) {
+                                op_ret = -1;
+                                op_errno = EINVAL;
+                                goto unlock;
+                        }
+
+                        if (local->retval)
+                                goto unlock;
+                        else if (volmark->retval) {
+                                GF_FREE (local->volmark);
+                                local->volmark =
+                                        memdup (volmark, sizeof (*volmark));
+                                local->retval = volmark->retval;
+                        } else if ((volmark->sec > local->volmark->sec) ||
+                                   ((volmark->sec == local->volmark->sec) &&
+                                    (volmark->usec >= local->volmark->usec))) {
+                                GF_FREE (local->volmark);
+                                local->volmark =
+                                      memdup (volmark, sizeof (*volmark));
+                        }
+
+                } else {
+                        local->volmark = memdup (volmark, sizeof (*volmark));
+                        VALIDATE_OR_GOTO (local->volmark, unlock);
+                        uuid_unparse (volmark->uuid, vol_uuid);
+                        if (volmark->retval)
+                                local->retval = volmark->retval;
+                        local->count[MCNT_FOUND]++;
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                op_ret = 0;
+                op_errno = 0;
+                need_unwind = 1;
+
+                if (local->count[MCNT_FOUND]) {
+                        if (!dict)
+                                dict = dict_new();
+
+                        if (dict_set_bin (dict, GF_XATTR_MARKER_KEY,
+                                          local->volmark,
+                                          sizeof (struct volume_mark))) {
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+                        }
+                }
+                op_errno = evaluate_marker_results (local->gauge, local->count);
+                if (op_errno)
+                        op_ret = -1;
+        }
+
+ out:
+        if (need_unwind && local && local->xl_specf_unwind) {
+                frame->local = local->xl_local;
+                local->xl_specf_unwind (frame, op_ret,
+                                        op_errno, dict, xdata);
+                return 0;
+        } else if (need_unwind){
+                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno,
+                                     dict, xdata);
+        }
+        return 0;
+}
+
+
+int32_t
+cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
+                       const char *name, void *xl_local,
+                       xlator_specf_unwind_t xl_specf_getxattr_unwind,
+                       xlator_t **sub_volumes, int count, int type,
+                       int *gauge, char *vol_uuid)
+{
+        int                i     = 0;
+        xl_marker_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (name, err);
+        VALIDATE_OR_GOTO (xl_specf_getxattr_unwind, err);
+
+        local = GF_CALLOC (sizeof (struct marker_str), 1,
+                            gf_common_mt_libxl_marker_local);
+
+        if (!local)
+                goto err;
+
+        local->xl_local = xl_local;
+        local->call_count = count;
+        local->xl_specf_unwind = xl_specf_getxattr_unwind;
+        local->vol_uuid = vol_uuid;
+        memcpy (local->gauge, gauge, sizeof (local->gauge));
+
+        frame->local = local;
+
+        for (i=0; i < count; i++) {
+                if (MARKER_UUID_TYPE == type)
+                        STACK_WIND (frame, cluster_markeruuid_cbk,
+                                    *(sub_volumes + i),
+                                    (*(sub_volumes + i))->fops->getxattr,
+                                    loc, name, NULL);
+                else if (MARKER_XTIME_TYPE == type)
+                        STACK_WIND (frame, cluster_markerxtime_cbk,
+                                    *(sub_volumes + i),
+                                    (*(sub_volumes + i))->fops->getxattr,
+                                    loc, name, NULL);
+                else {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                 "Unrecognized type (%d) of marker attr "
+                                 "received", type);
+                        STACK_WIND (frame, default_getxattr_cbk,
+                                    *(sub_volumes + i),
+                                    (*(sub_volumes + i))->fops->getxattr,
+                                    loc, name, NULL);
+                        break;
+                }
+        }
+
+        return 0;
+err:
+        return -1;
+
+}
+
+int
+gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value)
+{
+        int ret = -1;
+        uint32_t *net_timebuf = NULL;
+        uint32_t *value_timebuf = NULL;
+        uint32_t host_timebuf[2] = {0,};
+        uint32_t host_value_timebuf[2] = {0,};
+
+        /* stime should be minimum of all the other nodes */
+        ret = dict_get_bin (dst, key, (void **)&net_timebuf);
+        if (ret < 0) {
+                net_timebuf = GF_CALLOC (1, sizeof (int64_t),
+                                           gf_common_mt_char);
+                if (!net_timebuf)
+                        goto out;
+
+                ret = dict_set_bin (dst, key, net_timebuf, sizeof (int64_t));
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "key=%s: dict set failed", key);
+                        goto error;
+                }
+        }
+
+        value_timebuf = data_to_bin (value);
+        if (!value_timebuf) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "key=%s: getting value of stime failed", key);
+                ret = -1;
+                goto out;
+        }
+
+        get_hosttime (value_timebuf, host_value_timebuf);
+        get_hosttime (net_timebuf, host_timebuf);
+
+        /* can't use 'min()' macro here as we need to compare two fields
+           in the array, selectively */
+        if ((host_value_timebuf[0] > host_timebuf[0]) ||
+            ((host_value_timebuf[0] == host_timebuf[0]) &&
+             (host_value_timebuf[1] > host_timebuf[1]))) {
+                update_timebuf (value_timebuf, net_timebuf);
+        }
+
+        ret = 0;
+out:
+        return ret;
+error:
+        /* To be used only when net_timebuf is not set in the dict */
+        if (net_timebuf)
+                GF_FREE (net_timebuf);
+
+        return ret;
+}
diff --git a/xlators/lib/src/libxlator.h b/xlators/lib/src/libxlator.h
new file mode 100644
index 000000000..1d5e1657f
--- /dev/null
+++ b/xlators/lib/src/libxlator.h
@@ -0,0 +1,153 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _LIBXLATOR_H
+#define _LIBXLATOR_H
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+#include "common-utils.h"
+#include "compat.h"
+#include "compat-errno.h"
+
+
+#define MARKER_XATTR_PREFIX "trusted.glusterfs"
+#define XTIME               "xtime"
+#define VOLUME_MARK         "volume-mark"
+#define GF_XATTR_MARKER_KEY MARKER_XATTR_PREFIX "." VOLUME_MARK
+#define UUID_SIZE 36
+#define MARKER_UUID_TYPE    1
+#define MARKER_XTIME_TYPE   2
+#define GF_XATTR_QUOTA_SIZE_KEY "trusted.glusterfs.quota.size"
+
+
+typedef int32_t (*xlator_specf_unwind_t) (call_frame_t *frame,
+                                          int op_ret, int op_errno,
+                                          dict_t *dict, dict_t *xdata);
+
+
+struct volume_mark {
+        uint8_t major;
+        uint8_t minor;
+        uint8_t uuid[16];
+        uint8_t retval;
+        uint32_t sec;
+        uint32_t usec;
+}__attribute__ ((__packed__));
+
+
+/*
+ * The enumerated type here
+ * is used to index two kind
+ * of integer arrays:
+ * - gauges
+ * - counters
+
+ * A counter is used internally,
+ * in getxattr callbacks, to count
+ * the results, categorized as
+ * the enum names suggest. So values
+ * in the counter are always non-negative.
+
+ * Gauges are part of the API.
+ * The caller passes one to the
+ * top-level aggregator function,
+ * cluster_getmarkerattr(). The gauge
+ * defines an evaluation policy for the
+ * counter. That is, at the
+ * end of the aggregation process
+ * the gauge is matched against the
+ * counter, and the policy
+ * represented by the gauge decides
+ * whether to return with success or failure,
+ * and in latter case, what particular failure
+ * case (errno).
+
+ * The rules are the following: for some index i,
+ * - if gauge[i] == 0, no requirement is set
+ *   against counter[i];
+ * - if gauge[i] > 0, counter[i] >= gauge[i]
+ *   is required;
+ * - if gauge[i] < 0, counter[i] < |gauge[i]|
+ *   is required.
+
+ * If the requirement is not met, then i is mapped
+ * to the respective errno (MCNT_ENOENT -> ENOENT),
+ * or in lack of that, EINVAL.
+
+ * Cf. evaluate_marker_results() and marker_idx_errno_map[]
+ * in libxlator.c
+
+ * We provide two default gauges, one inteded for xtime
+ * aggregation, other for volume mark aggregation. The
+ * policies they represent agree with the hard-coded
+ * one prior to gauges. Cf. marker_xtime_default_gauge
+ * and marker_uuid_default_gauge in libxlator.c
+ */
+
+typedef enum {
+        MCNT_FOUND,
+        MCNT_NOTFOUND,
+        MCNT_ENODATA,
+        MCNT_ENOTCONN,
+        MCNT_ENOENT,
+        MCNT_EOTHER,
+        MCNT_MAX
+} marker_result_idx_t;
+
+extern int marker_xtime_default_gauge[];
+extern int marker_uuid_default_gauge[];
+
+struct marker_str {
+        struct volume_mark    *volmark;
+        data_t                *data;
+
+        uint32_t               host_timebuf[2];
+        uint32_t               net_timebuf[2];
+        int32_t                call_count;
+        int                    gauge[MCNT_MAX];
+        int                    count[MCNT_MAX];
+
+        xlator_specf_unwind_t  xl_specf_unwind;
+        void                  *xl_local;
+        char                  *vol_uuid;
+        uint8_t                retval;
+};
+
+typedef struct marker_str xl_marker_local_t;
+
+int32_t
+cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
+
+int32_t
+cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
+
+int32_t
+cluster_getmarkerattr (call_frame_t *frame,xlator_t *this, loc_t *loc,
+                       const char *name, void *xl_local,
+                       xlator_specf_unwind_t xl_specf_getxattr_unwind,
+                       xlator_t **sub_volumes, int count, int type,
+                       int *gauge, char *vol_uuid);
+
+int
+match_uuid_local (const char *name, char *uuid);
+
+int
+gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value);
+
+#endif /* !_LIBXLATOR_H */
diff --git a/xlators/meta/src/Makefile.am b/xlators/meta/src/Makefile.am
index 385ff553f..f8fa7d4cb 100644
--- a/xlators/meta/src/Makefile.am
+++ b/xlators/meta/src/Makefile.am
@@ -4,7 +4,8 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/
 meta_so_SOURCES = meta.c tree.c misc.c view.c
 noinst_HEADERS = meta.h tree.h misc.h view.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall
 
 CLEANFILES = 
diff --git a/xlators/meta/src/meta-mem-types.h b/xlators/meta/src/meta-mem-types.h
new file mode 100644
index 000000000..62028b246
--- /dev/null
+++ b/xlators/meta/src/meta-mem-types.h
@@ -0,0 +1,25 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __META_MEM_TYPES_H__
+#define __META_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_meta_mem_types_ {
+        gf_meta_mt__open_local = gf_common_mt_end + 1,
+        gf_meta_mt_dir_entry_t,
+        gf_meta_mt_meta_dirent_t,
+        gf_meta_mt_meta_private_t,
+        gf_meta_mt_stat,
+        gf_meta_mt_end
+};
+#endif
+
diff --git a/xlators/meta/src/meta.c b/xlators/meta/src/meta.c
index ce49ed2c4..e69719f3c 100644
--- a/xlators/meta/src/meta.c
+++ b/xlators/meta/src/meta.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -32,6 +22,7 @@
 
 #include "meta.h"
 #include "view.h"
+#include "meta-mem-types.h"
 
 int32_t
 meta_getattr_cbk (call_frame_t *frame,
@@ -516,9 +507,9 @@ meta_open (call_frame_t *frame, xlator_t *this,
 
   if (file) {
     if (file->fops && file->fops->open) {
-      struct _open_local *local = CALLOC (1, sizeof (struct _open_local));
+            struct _open_local *local = GF_CALLOC (1, sizeof (struct _open_local), gf_meta_mt__open_local);
       ERR_ABORT (local);
-      local->path = strdup (path);
+      local->path = gf_strdup (path);
       frame->local = local;
       STACK_WIND (frame, meta_open_cbk,
 		  this, file->fops->open,
@@ -528,7 +519,7 @@ meta_open (call_frame_t *frame, xlator_t *this,
     else {
       dict_t *ctx = get_new_dict ();
       dict_ref (ctx);
-      dict_set (ctx, this->name, str_to_data (strdup (path)));
+      dict_set (ctx, this->name, str_to_data (gf_strdup (path)));
       STACK_UNWIND (frame, 0, 0, ctx, file->stbuf);
       return 0;
     }
@@ -551,9 +542,9 @@ meta_create (call_frame_t *frame, xlator_t *this,
 
   if (file) {
     if (file->fops && file->fops->create) {
-      struct _open_local *local = CALLOC (1, sizeof (struct _open_local));
+            struct _open_local *local = GF_CALLOC (1, sizeof (struct _open_local), gf_meta_mt__open_local);
       ERR_ABORT (local);
-      local->path = strdup (path);
+      local->path = gf_strdup (path);
       frame->local = local;
       STACK_WIND (frame, meta_open_cbk,
 		  this, file->fops->create,
@@ -826,7 +817,7 @@ meta_opendir (call_frame_t *frame,
   
   if (dir) {
     dict_t *ctx = get_new_dict ();
-    dict_set (ctx, this->name, str_to_data (strdup (path)));
+    dict_set (ctx, this->name, str_to_data (gf_strdup (path)));
     STACK_UNWIND (frame, 0, 0, ctx);
     return 0;
   }
@@ -850,10 +841,11 @@ meta_readdir_cbk (call_frame_t *frame,
   meta_private_t *priv = (meta_private_t *)this->private;
 
   if ((int) cookie == 1) {
-    dir_entry_t *dir = CALLOC (1, sizeof (dir_entry_t));
+    dir_entry_t *dir = GF_CALLOC (1, sizeof (dir_entry_t),
+                                  gf_meta_mt_dir_entry_t);
     ERR_ABORT (dir);
 
-    dir->name = strdup (".meta");
+    dir->name = gf_strdup (".meta");
     memcpy (&dir->buf, priv->tree->stbuf, sizeof (struct stat));
     dir->next = entries->next;
     entries->next = dir;
@@ -887,7 +879,8 @@ meta_readdir (call_frame_t *frame,
       dir_entry_t *entries = NULL;
 
       while (dir) {
-	dir_entry_t *d = CALLOC (1, sizeof (dir_entry_t));
+        dir_entry_t *d = GF_CALLOC (1, sizeof (dir_entry_t),
+                                    gf_meta_mt_dir_entry_t);
 	ERR_ABORT (d);
 	d->name = dir->name;
 	d->buf  = *dir->stbuf;
@@ -897,7 +890,8 @@ meta_readdir (call_frame_t *frame,
 	dir = dir->next;
       }
 
-      dir_entry_t *header = CALLOC (1, sizeof (dir_entry_t));
+      dir_entry_t *header = GF_CALLOC (1, sizeof (dir_entry_t),
+                                       gf_meta_mt_dir_entry_t);
       ERR_ABORT (header);
       header->next = entries;
       STACK_UNWIND (frame, 0, 0, header, count);
@@ -1132,7 +1126,7 @@ meta_lk_cbk (call_frame_t *frame,
 	     xlator_t *this,
 	     int32_t op_ret,
 	     int32_t op_errno,
-	     struct flock *lock)
+	     struct gf_flock *lock)
 {
   STACK_UNWIND (frame,
 		op_ret,
@@ -1146,7 +1140,7 @@ meta_lk (call_frame_t *frame,
 	 xlator_t *this,
 	 dict_t *file,
 	 int32_t cmd,
-	 struct flock *lock)
+	 struct gf_flock *lock)
 {
   STACK_WIND (frame,
 	      meta_lk_cbk,
@@ -1163,16 +1157,16 @@ add_xlator_to_tree (meta_dirent_t *tree, xlator_t *this,
 		    const char *prefix)
 {
   char *dir;
-  asprintf (&dir, "%s/%s", prefix, this->name);
+  gf_asprintf (&dir, "%s/%s", prefix, this->name);
 
   char *children;
-  asprintf (&children, "%s/%s", dir, "subvolumes");
+  gf_asprintf (&children, "%s/%s", dir, "subvolumes");
 
   char *type;
-  asprintf (&type, "%s/%s", dir, "type");
+  gf_asprintf (&type, "%s/%s", dir, "type");
 
   char *view;
-  asprintf (&view, "%s/%s", dir, "view");
+  gf_asprintf (&view, "%s/%s", dir, "view");
 
   insert_meta_entry (tree, dir, S_IFDIR, NULL, NULL);
   insert_meta_entry (tree, children, S_IFDIR, NULL, NULL);
@@ -1194,9 +1188,10 @@ static void
 build_meta_tree (xlator_t *this)
 {
   meta_private_t *priv = (meta_private_t *) this->private;
-  priv->tree = CALLOC (1, sizeof (meta_dirent_t));
+  priv->tree = GF_CALLOC (1, sizeof (meta_dirent_t),
+                          gf_meta_mt_meta_dirent_t);
   ERR_ABORT (priv->tree);
-  priv->tree->name = strdup (".meta");
+  priv->tree->name = gf_strdup (".meta");
   priv->tree->stbuf = new_stbuf ();
   priv->tree->stbuf->st_mode = S_IFDIR | S_IRUSR | S_IRGRP | S_IROTH |
     S_IXUSR | S_IXGRP | S_IXOTH;
@@ -1215,6 +1210,25 @@ build_meta_tree (xlator_t *this)
 }
 
 int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_meta_mt_end + 1);
+        
+        if (ret != 0) {
+                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+                       "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
 init (xlator_t *this)
 {
   if (this->parent != NULL) {
@@ -1222,12 +1236,13 @@ init (xlator_t *this)
     return -1;
   }
   
-  meta_private_t *priv = CALLOC (1, sizeof (meta_private_t));
+  meta_private_t *priv = GF_CALLOC (1, sizeof (meta_private_t),
+                                    gf_meta_mt_meta_private_t);
   ERR_ABORT (priv);
   
   data_t *directory = dict_get (this->options, "directory");
   if (directory) {
-    priv->directory = strdup (data_to_str (directory));
+    priv->directory = gf_strdup (data_to_str (directory));
   }
   else {
     priv->directory = ".meta";
@@ -1280,6 +1295,3 @@ struct xlator_fops fops = {
   .create      = meta_create,
   .lk          = meta_lk,
 };
-
-struct xlator_mops mops = {
-};
diff --git a/xlators/meta/src/meta.h b/xlators/meta/src/meta.h
index 6823ef85b..73e0e50db 100644
--- a/xlators/meta/src/meta.h
+++ b/xlators/meta/src/meta.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __META_H__
 #define __META_H__
 
diff --git a/xlators/meta/src/misc.c b/xlators/meta/src/misc.c
index 9c2f50d34..1a8dfa806 100644
--- a/xlators/meta/src/misc.c
+++ b/xlators/meta/src/misc.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #include <unistd.h>
 #include <sys/uio.h>
 
diff --git a/xlators/meta/src/misc.h b/xlators/meta/src/misc.h
index 433c604eb..30dd10e34 100644
--- a/xlators/meta/src/misc.h
+++ b/xlators/meta/src/misc.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MISC_H__
 #define __MISC_H__
 
diff --git a/xlators/meta/src/tree.c b/xlators/meta/src/tree.c
index ec88c42a0..dacbd665a 100644
--- a/xlators/meta/src/tree.c
+++ b/xlators/meta/src/tree.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -31,6 +21,7 @@
 #include "xlator.h"
 
 #include "meta.h"
+#include "meta-mem-types.h"
 
 static int
 is_meta_path (const char *path)
@@ -46,7 +37,7 @@ struct stat *
 new_stbuf (void)
 {
   static int next_inode = 0;
-  struct stat *stbuf = CALLOC (1, sizeof (struct stat));
+  struct stat *stbuf = GF_CALLOC (1, sizeof (struct stat), gf_meta_mt_stat);
   
   ERR_ABORT (stbuf);
 
@@ -99,7 +90,7 @@ meta_dirent_t *
 lookup_meta_entry (meta_dirent_t *root, const char *path,
 		   char **remain)
 {
-  char *_path = strdup (path);
+  char *_path = gf_strdup (path);
 
   if (!is_meta_path (path))
     return NULL;
@@ -119,10 +110,10 @@ lookup_meta_entry (meta_dirent_t *root, const char *path,
 	while (piece) {
 	  char *tmp = *remain;
 	  if (*remain)
-	    asprintf (remain, "/%s/%s", *remain, piece);
+	    gf_asprintf (remain, "/%s/%s", *remain, piece);
 	  else
-	    asprintf (remain, "/%s", piece);
-	  if (tmp) free (tmp);
+	    gf_asprintf (remain, "/%s", piece);
+	  GF_FREE (tmp);
 	  piece = strtok (NULL, "/");
 	}
       }
@@ -132,7 +123,7 @@ lookup_meta_entry (meta_dirent_t *root, const char *path,
     trav = ntrav;
   }
 
-  free (_path);
+  GF_FREE (_path);
   return trav;
 }
 
@@ -148,9 +139,10 @@ insert_meta_entry (meta_dirent_t *root, const char *path,
   if (!dir)
     return NULL;
 
-  meta_dirent_t *new = CALLOC (1, sizeof (meta_dirent_t));
+  meta_dirent_t *new = GF_CALLOC (1, sizeof (meta_dirent_t),
+                                  gf_meta_mt_meta_dirent_t);
   ERR_ABORT (new);
-  new->name        = strdup (slashpos+1);
+  new->name        = gf_strdup (slashpos+1);
   new->type        = type;
   new->parent      = parent;
   new->next        = parent->children;
@@ -167,9 +159,10 @@ insert_meta_entry (meta_dirent_t *root, const char *path,
 
 int main (void)
 {
-  meta_dirent_t *root = CALLOC (1, sizeof (meta_dirent_t));
+  meta_dirent_t *root = GF_CALLOC (1, sizeof (meta_dirent_t),
+                                   gf_meta_mt_meta_dirent_t);
   ERR_ABORT (root);
-  root->name = strdup (".meta");
+  root->name = gf_strdup (".meta");
 
   insert_meta_entry (root, "/.meta/version", S_IFREG, NULL, NULL);
   return 0;
diff --git a/xlators/meta/src/tree.h b/xlators/meta/src/tree.h
index eb2cf0220..985df3bd7 100644
--- a/xlators/meta/src/tree.h
+++ b/xlators/meta/src/tree.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __TREE_H__
 #define __TREE_H__
 
diff --git a/xlators/meta/src/view.c b/xlators/meta/src/view.c
index 7104d10e9..b4e2d64a2 100644
--- a/xlators/meta/src/view.c
+++ b/xlators/meta/src/view.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/meta/src/view.h b/xlators/meta/src/view.h
index 2e1ac3ebf..2eff6126e 100644
--- a/xlators/meta/src/view.h
+++ b/xlators/meta/src/view.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __VIEW_H__
 #define __VIEW_H__
 
diff --git a/xlators/mgmt/Makefile.am b/xlators/mgmt/Makefile.am
new file mode 100644
index 000000000..bf09b07c3
--- /dev/null
+++ b/xlators/mgmt/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = glusterd
+
+CLEANFILES = 
diff --git a/xlators/mgmt/glusterd/Makefile.am b/xlators/mgmt/glusterd/Makefile.am
new file mode 100644
index 000000000..d471a3f92
--- /dev/null
+++ b/xlators/mgmt/glusterd/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES = 
diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am
new file mode 100644
index 000000000..933c44019
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/Makefile.am
@@ -0,0 +1,50 @@
+xlator_LTLIBRARIES = glusterd.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mgmt
+glusterd_la_CPPFLAGS = $(AM_CPPFLAGS) "-DFILTERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/filter\""
+glusterd_la_LDFLAGS = -module -avoid-version
+if ENABLE_BD_XLATOR
+glusterd_la_LDFLAGS += -llvm2app
+endif
+glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \
+	glusterd-op-sm.c glusterd-utils.c glusterd-rpc-ops.c \
+	glusterd-store.c glusterd-handshake.c glusterd-pmap.c \
+	glusterd-volgen.c glusterd-rebalance.c glusterd-quota.c \
+	glusterd-geo-rep.c glusterd-replace-brick.c glusterd-log-ops.c \
+	glusterd-volume-ops.c glusterd-brick-ops.c glusterd-mountbroker.c \
+	glusterd-syncop.c glusterd-hooks.c glusterd-volume-set.c \
+        glusterd-locks.c glusterd-snapshot.c glusterd-mgmt-handler.c \
+        glusterd-mgmt.c
+
+glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+		     $(top_builddir)/rpc/xdr/src/libgfxdr.la \
+		     $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+		     $(XML_LIBS) -lcrypto
+
+noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \
+	glusterd-sm.h glusterd-store.h glusterd-mem-types.h \
+	glusterd-pmap.h glusterd-volgen.h glusterd-mountbroker.h \
+	glusterd-syncop.h glusterd-hooks.h glusterd-locks.h \
+	glusterd-mgmt.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(rpclibdir) -I$(CONTRIBDIR)/rbtree \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(CONTRIBDIR)/uuid \
+	-DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \
+	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\"\
+        -DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(XML_CPPFLAGS)
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+AM_LDFLAGS = -L$(xlatordir)
+
+CLEANFILES =
+
+install-data-hook:
+
+if GF_INSTALL_VAR_LIB_GLUSTERD
+	$(mkdir_p) $(localstatedir)/lib/
+	(stat $(sysconfdir)/glusterd && \
+	    mv $(sysconfdir)/glusterd $(localstatedir)/lib/) || true;
+	(ln -sf $(localstatedir)/lib/glusterd $(sysconfdir)/glusterd) || true;
+endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
new file mode 100644
index 000000000..596503c21
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -0,0 +1,1953 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "common-utils.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "run.h"
+#include <sys/signal.h>
+
+/* misc */
+
+/* In this function, we decide, based on the 'count' of the brick,
+   where to add it in the current volume. 'count' tells us already
+   how many of the given bricks are added. other argument are self-
+   descriptive. */
+int
+add_brick_at_right_order (glusterd_brickinfo_t *brickinfo,
+                          glusterd_volinfo_t *volinfo, int count,
+                          int32_t stripe_cnt, int32_t replica_cnt)
+{
+        int                   idx     = 0;
+        int                   i       = 0;
+        int                   sub_cnt = 0;
+        glusterd_brickinfo_t *brick   = NULL;
+
+        /* The complexity of the function is in deciding at which index
+           to add new brick. Even though it can be defined with a complex
+           single formula for all volume, it is seperated out to make it
+           more readable */
+        if (stripe_cnt) {
+                /* common formula when 'stripe_count' is set */
+                /* idx = ((count / ((stripe_cnt * volinfo->replica_count) -
+                   volinfo->dist_leaf_count)) * volinfo->dist_leaf_count) +
+                   (count + volinfo->dist_leaf_count);
+                */
+
+                sub_cnt = volinfo->dist_leaf_count;
+
+                idx = ((count / ((stripe_cnt * volinfo->replica_count) -
+                                 sub_cnt)) * sub_cnt) +
+                        (count + sub_cnt);
+
+                goto insert_brick;
+        }
+
+        /* replica count is set */
+        /* common formula when 'replica_count' is set */
+        /* idx = ((count / (replica_cnt - existing_replica_count)) *
+           existing_replica_count) +
+           (count + existing_replica_count);
+        */
+
+        sub_cnt = volinfo->replica_count;
+        idx = (count / (replica_cnt - sub_cnt) * sub_cnt) +
+                (count + sub_cnt);
+
+insert_brick:
+        i = 0;
+        list_for_each_entry (brick, &volinfo->bricks, brick_list) {
+                i++;
+                if (i < idx)
+                        continue;
+                gf_log (THIS->name, GF_LOG_DEBUG, "brick:%s index=%d, count=%d",
+                        brick->path, idx, count);
+
+                list_add (&brickinfo->brick_list, &brick->brick_list);
+                break;
+        }
+
+        return 0;
+}
+
+
+static int
+gd_addbr_validate_stripe_count (glusterd_volinfo_t *volinfo, int stripe_count,
+                                int total_bricks, int *type, char *err_str,
+                                size_t err_len)
+{
+        int ret = -1;
+
+        switch (volinfo->type) {
+        case GF_CLUSTER_TYPE_NONE:
+                if ((volinfo->brick_count * stripe_count) == total_bricks) {
+                        /* Change the volume type */
+                        *type = GF_CLUSTER_TYPE_STRIPE;
+                        gf_log (THIS->name, GF_LOG_INFO,
+                                "Changing the type of volume %s from "
+                                "'distribute' to 'stripe'", volinfo->volname);
+                        ret = 0;
+                        goto out;
+                } else {
+                        snprintf (err_str, err_len, "Incorrect number of "
+                                  "bricks (%d) supplied for stripe count (%d).",
+                                  (total_bricks - volinfo->brick_count),
+                                  stripe_count);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+                break;
+        case GF_CLUSTER_TYPE_REPLICATE:
+                if (!(total_bricks % (volinfo->replica_count * stripe_count))) {
+                        /* Change the volume type */
+                        *type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
+                        gf_log (THIS->name, GF_LOG_INFO,
+                                "Changing the type of volume %s from "
+                                "'replicate' to 'replicate-stripe'",
+                                volinfo->volname);
+                        ret = 0;
+                        goto out;
+                } else {
+                        snprintf (err_str, err_len, "Incorrect number of "
+                                  "bricks (%d) supplied for changing volume's "
+                                  "stripe count to %d, need at least %d bricks",
+                                  (total_bricks - volinfo->brick_count),
+                                  stripe_count,
+                                  (volinfo->replica_count * stripe_count));
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+                break;
+        case GF_CLUSTER_TYPE_STRIPE:
+        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                if (stripe_count < volinfo->stripe_count) {
+                        snprintf (err_str, err_len,
+                                  "Incorrect stripe count (%d) supplied. "
+                                  "Volume already has stripe count (%d)",
+                                  stripe_count, volinfo->stripe_count);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+                if (stripe_count == volinfo->stripe_count) {
+                        if (!(total_bricks % volinfo->dist_leaf_count)) {
+                                /* its same as the one which exists */
+                                ret = 1;
+                                goto out;
+                        }
+                }
+                if (stripe_count > volinfo->stripe_count) {
+                        /* We have to make sure before and after 'add-brick',
+                           the number or subvolumes for distribute will remain
+                           same, when stripe count is given */
+                        if ((volinfo->brick_count * (stripe_count *
+                                                     volinfo->replica_count)) ==
+                            (total_bricks * volinfo->dist_leaf_count)) {
+                                /* Change the dist_leaf_count */
+                                gf_log (THIS->name, GF_LOG_INFO,
+                                        "Changing the stripe count of "
+                                        "volume %s from %d to %d",
+                                        volinfo->volname,
+                                        volinfo->stripe_count, stripe_count);
+                                ret = 0;
+                                goto out;
+                        }
+                }
+                break;
+        }
+
+out:
+        return ret;
+}
+
+static int
+gd_addbr_validate_replica_count (glusterd_volinfo_t *volinfo, int replica_count,
+                                 int total_bricks, int *type, char *err_str,
+                                 int err_len)
+{
+        int ret = -1;
+
+        /* replica count is set */
+        switch (volinfo->type) {
+        case GF_CLUSTER_TYPE_NONE:
+                if ((volinfo->brick_count * replica_count) == total_bricks) {
+                        /* Change the volume type */
+                        *type = GF_CLUSTER_TYPE_REPLICATE;
+                        gf_log (THIS->name, GF_LOG_INFO,
+                                "Changing the type of volume %s from "
+                                "'distribute' to 'replica'", volinfo->volname);
+                        ret = 0;
+                        goto out;
+
+                } else {
+                        snprintf (err_str, err_len, "Incorrect number of "
+                                  "bricks (%d) supplied for replica count (%d).",
+                                  (total_bricks - volinfo->brick_count),
+                                  replica_count);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+                break;
+        case GF_CLUSTER_TYPE_STRIPE:
+                if (!(total_bricks % (volinfo->dist_leaf_count * replica_count))) {
+                        /* Change the volume type */
+                        *type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
+                        gf_log (THIS->name, GF_LOG_INFO,
+                                "Changing the type of volume %s from "
+                                "'stripe' to 'replicate-stripe'",
+                                volinfo->volname);
+                        ret = 0;
+                        goto out;
+                } else {
+                        snprintf (err_str, err_len, "Incorrect number of "
+                                  "bricks (%d) supplied for changing volume's "
+                                  "replica count to %d, need at least %d "
+                                  "bricks",
+                                  (total_bricks - volinfo->brick_count),
+                                  replica_count, (volinfo->dist_leaf_count *
+                                                  replica_count));
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+                break;
+        case GF_CLUSTER_TYPE_REPLICATE:
+        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                if (replica_count < volinfo->replica_count) {
+                        snprintf (err_str, err_len,
+                                  "Incorrect replica count (%d) supplied. "
+                                  "Volume already has (%d)",
+                                  replica_count, volinfo->replica_count);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+                if (replica_count == volinfo->replica_count) {
+                        if (!(total_bricks % volinfo->dist_leaf_count)) {
+                                ret = 1;
+                                goto out;
+                        }
+                }
+                if (replica_count > volinfo->replica_count) {
+                        /* We have to make sure before and after 'add-brick',
+                           the number or subvolumes for distribute will remain
+                           same, when replica count is given */
+                        if ((total_bricks * volinfo->dist_leaf_count) ==
+                            (volinfo->brick_count * (replica_count *
+                                                     volinfo->stripe_count))) {
+                                /* Change the dist_leaf_count */
+                                gf_log (THIS->name, GF_LOG_INFO,
+                                        "Changing the replica count of "
+                                        "volume %s from %d to %d",
+                                        volinfo->volname, volinfo->replica_count,
+                                        replica_count);
+                                ret = 0;
+                                goto out;
+                        }
+                }
+                break;
+        }
+out:
+        return ret;
+}
+
+static int
+gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo,
+                                int32_t replica_count,
+                                int32_t brick_count, char *err_str,
+                                size_t err_len)
+{
+        int ret = -1;
+        int replica_nodes = 0;
+
+        switch (volinfo->type) {
+        case GF_CLUSTER_TYPE_NONE:
+        case GF_CLUSTER_TYPE_STRIPE:
+                snprintf (err_str, err_len,
+                          "replica count (%d) option given for non replicate "
+                          "volume %s", replica_count, volinfo->volname);
+                gf_log (THIS->name, GF_LOG_WARNING, "%s", err_str);
+                goto out;
+
+        case GF_CLUSTER_TYPE_REPLICATE:
+        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                /* in remove brick, you can only reduce the replica count */
+                if (replica_count > volinfo->replica_count) {
+                        snprintf (err_str, err_len,
+                                  "given replica count (%d) option is more "
+                                  "than volume %s's replica count (%d)",
+                                  replica_count, volinfo->volname,
+                                  volinfo->replica_count);
+                        gf_log (THIS->name, GF_LOG_WARNING, "%s", err_str);
+                        goto out;
+                }
+                if (replica_count == volinfo->replica_count) {
+                        /* This means the 'replica N' option on CLI was
+                           redundant. Check if the total number of bricks given
+                           for removal is same as 'dist_leaf_count' */
+                        if (brick_count % volinfo->dist_leaf_count) {
+                                snprintf (err_str, err_len,
+                                          "number of bricks provided (%d) is "
+                                          "not valid. need at least %d "
+                                          "(or %dxN)", brick_count,
+                                          volinfo->dist_leaf_count,
+                                          volinfo->dist_leaf_count);
+                                gf_log (THIS->name, GF_LOG_WARNING, "%s",
+                                        err_str);
+                                goto out;
+                        }
+                        ret = 1;
+                        goto out;
+                }
+
+                replica_nodes = ((volinfo->brick_count /
+                                  volinfo->replica_count) *
+                                 (volinfo->replica_count - replica_count));
+
+                if (brick_count % replica_nodes) {
+                        snprintf (err_str, err_len,
+                                  "need %d(xN) bricks for reducing replica "
+                                  "count of the volume from %d to %d",
+                                  replica_nodes, volinfo->replica_count,
+                                  replica_count);
+                        goto out;
+                }
+                break;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/* Handler functions */
+int
+__glusterd_handle_add_brick (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        dict_t                          *dict = NULL;
+        char                            *bricks = NULL;
+        char                            *volname = NULL;
+        int                             brick_count = 0;
+        void                            *cli_rsp = NULL;
+        char                            err_str[2048] = {0,};
+        gf_cli_rsp                      rsp = {0,};
+        glusterd_volinfo_t              *volinfo = NULL;
+        xlator_t                        *this = NULL;
+        int                             total_bricks = 0;
+        int32_t                         replica_count = 0;
+        int32_t                         stripe_count = 0;
+        int                             type = 0;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                snprintf (err_str, sizeof (err_str), "Garbage args received");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Received add brick req");
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        if (!(ret = glusterd_check_volume_exists (volname))) {
+                ret = -1;
+                snprintf (err_str, sizeof (err_str), "Volume %s does not exist",
+                          volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &brick_count);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "brick count");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "replica-count", &replica_count);
+        if (!ret) {
+                gf_log (this->name, GF_LOG_INFO, "replica-count is %d",
+                        replica_count);
+        }
+
+        ret = dict_get_int32 (dict, "stripe-count", &stripe_count);
+        if (!ret) {
+                gf_log (this->name, GF_LOG_INFO, "stripe-count is %d",
+                        stripe_count);
+        }
+
+        if (!dict_get (dict, "force")) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get flag");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get volinfo "
+                          "for volume name %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+
+        }
+
+        total_bricks = volinfo->brick_count + brick_count;
+
+        if (!stripe_count && !replica_count) {
+                if (volinfo->type == GF_CLUSTER_TYPE_NONE)
+                        goto brick_val;
+
+                if ((volinfo->brick_count < volinfo->dist_leaf_count) &&
+                    (total_bricks <= volinfo->dist_leaf_count))
+                        goto brick_val;
+
+                if ((brick_count % volinfo->dist_leaf_count) != 0) {
+                        snprintf (err_str, sizeof (err_str), "Incorrect number "
+                                  "of bricks supplied %d with count %d",
+                                 brick_count, volinfo->dist_leaf_count);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                        ret = -1;
+                        goto out;
+                }
+                goto brick_val;
+                /* done with validation.. below section is if stripe|replica
+                   count is given */
+        }
+
+        /* These bricks needs to be added one per a replica or stripe volume */
+        if (stripe_count) {
+                ret = gd_addbr_validate_stripe_count (volinfo, stripe_count,
+                                                      total_bricks, &type,
+                                                      err_str,
+                                                      sizeof (err_str));
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+
+                /* if stripe count is same as earlier, set it back to 0 */
+                if (ret == 1)
+                        stripe_count = 0;
+
+                ret = dict_set_int32 (dict, "stripe-count", stripe_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to set the stripe-count in dict");
+                        goto out;
+                }
+                goto brick_val;
+        }
+
+        ret = gd_addbr_validate_replica_count (volinfo, replica_count,
+                                               total_bricks,
+                                               &type, err_str,
+                                               sizeof (err_str));
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        /* if replica count is same as earlier, set it back to 0 */
+        if (ret == 1)
+                replica_count = 0;
+
+        ret = dict_set_int32 (dict, "replica-count", replica_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set the replica-count in dict");
+                goto out;
+        }
+
+brick_val:
+        ret = dict_get_str (dict, "bricks", &bricks);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "bricks");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        if (type != volinfo->type) {
+                ret = dict_set_int32 (dict, "type", type);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to set the new type in dict");
+        }
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
+
+out:
+        if (ret) {
+                rsp.op_ret = -1;
+                rsp.op_errno = 0;
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str), "Operation failed");
+                rsp.op_errstr = err_str;
+                cli_rsp = &rsp;
+                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
+                ret = 0; //sent error to cli, prevent second reply
+        }
+
+        free (cli_req.dict.dict_val); //its malloced by xdr
+
+        return ret;
+}
+
+int
+glusterd_handle_add_brick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_add_brick);
+}
+
+static int
+subvol_matcher_init (int **subvols, int count)
+{
+        int ret = -1;
+
+        *subvols = GF_CALLOC (count, sizeof(int), gf_gld_mt_int);
+        if (*subvols)
+                ret = 0;
+
+        return ret;
+}
+
+static void
+subvol_matcher_update (int *subvols, glusterd_volinfo_t *volinfo,
+                       glusterd_brickinfo_t *brickinfo)
+{
+        glusterd_brickinfo_t *tmp        = NULL;
+        int32_t               sub_volume = 0;
+        int                   pos        = 0;
+
+        list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
+
+                if (strcmp (tmp->hostname, brickinfo->hostname) ||
+                    strcmp (tmp->path, brickinfo->path)) {
+                        pos++;
+                        continue;
+                }
+                gf_log (THIS->name, GF_LOG_DEBUG, LOGSTR_FOUND_BRICK,
+                                        brickinfo->hostname, brickinfo->path,
+                                        volinfo->volname);
+                sub_volume = (pos / volinfo->dist_leaf_count);
+                subvols[sub_volume]++;
+                break;
+        }
+
+}
+
+static int
+subvol_matcher_verify (int *subvols, glusterd_volinfo_t *volinfo, char *err_str,
+                       size_t err_len, char *vol_type)
+{
+        int i = 0;
+        int ret = 0;
+
+       do {
+
+                if (subvols[i] % volinfo->dist_leaf_count == 0) {
+                        continue;
+                } else {
+                        ret = -1;
+                        snprintf (err_str, err_len,
+                                "Bricks not from same subvol for %s", vol_type);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
+                        break;
+                }
+        } while (++i < volinfo->subvol_count);
+
+        return ret;
+}
+
+static void
+subvol_matcher_destroy (int *subvols)
+{
+        GF_FREE (subvols);
+}
+
+int
+__glusterd_handle_remove_brick (rpcsvc_request_t *req)
+{
+        int32_t                   ret              = -1;
+        gf_cli_req                cli_req          = {{0,}};
+        dict_t                   *dict             = NULL;
+        int32_t                   count            = 0;
+        char                     *brick            = NULL;
+        char                      key[256]         = {0,};
+        char                     *brick_list       = NULL;
+        int                       i                = 1;
+        glusterd_volinfo_t       *volinfo          = NULL;
+        glusterd_brickinfo_t     *brickinfo        = NULL;
+        int                      *subvols          = NULL;
+        glusterd_brickinfo_t     *tmp              = NULL;
+        char                      err_str[2048]    = {0};
+        gf_cli_rsp                rsp              = {0,};
+        void                     *cli_rsp          = NULL;
+        char                      vol_type[256]    = {0,};
+        int32_t                   replica_count    = 0;
+        int32_t                   brick_index      = 0;
+        int32_t                   tmp_brick_idx    = 0;
+        int                       found            = 0;
+        int                       diff_count       = 0;
+        char                     *volname          = 0;
+        xlator_t                 *this             = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                snprintf (err_str, sizeof (err_str), "Received garbage args");
+                goto out;
+        }
+
+
+        gf_log (this->name, GF_LOG_INFO, "Received rem brick req");
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get brick "
+                          "count");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                 snprintf (err_str, sizeof (err_str),"Volume %s does not exist",
+                           volname);
+                 gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                 goto out;
+        }
+
+        ret = dict_get_int32 (dict, "replica-count", &replica_count);
+        if (!ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "request to change replica-count to %d", replica_count);
+                ret = gd_rmbr_validate_replica_count (volinfo, replica_count,
+                                                      count, err_str,
+                                                      sizeof (err_str));
+                if (ret < 0) {
+                        /* logging and error msg are done in above function
+                           itself */
+                        goto out;
+                }
+                dict_del (dict, "replica-count");
+                if (ret) {
+                        replica_count = 0;
+                } else {
+                        ret = dict_set_int32 (dict, "replica-count",
+                                              replica_count);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to set the replica_count "
+                                        "in dict");
+                                goto out;
+                        }
+                }
+        }
+
+        /* 'vol_type' is used for giving the meaning full error msg for user */
+        if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
+                strcpy (vol_type, "replica");
+        } else if (volinfo->type == GF_CLUSTER_TYPE_STRIPE) {
+                strcpy (vol_type, "stripe");
+        } else if (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) {
+                strcpy (vol_type, "stripe-replicate");
+        } else {
+                strcpy (vol_type, "distribute");
+        }
+
+	/* Do not allow remove-brick if the volume is a stripe volume*/
+	if ((volinfo->type == GF_CLUSTER_TYPE_STRIPE) &&
+            (volinfo->brick_count == volinfo->stripe_count)) {
+                snprintf (err_str, sizeof (err_str),
+                          "Removing brick from a stripe volume is not allowed");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+	}
+
+	if (!replica_count &&
+            (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) &&
+            (volinfo->brick_count == volinfo->dist_leaf_count)) {
+                snprintf (err_str, sizeof(err_str),
+                          "Removing bricks from stripe-replicate"
+                          " configuration is not allowed without reducing "
+                          "replica or stripe count explicitly.");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+	if (!replica_count &&
+            (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+            (volinfo->brick_count == volinfo->dist_leaf_count)) {
+                snprintf (err_str, sizeof (err_str),
+                          "Removing bricks from replicate configuration "
+                          "is not allowed without reducing replica count "
+                          "explicitly.");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+	/* Do not allow remove-brick if the bricks given is less than
+           the replica count or stripe count */
+        if (!replica_count && (volinfo->type != GF_CLUSTER_TYPE_NONE)) {
+                if (volinfo->dist_leaf_count &&
+                    (count % volinfo->dist_leaf_count)) {
+                        snprintf (err_str, sizeof (err_str), "Remove brick "
+                                  "incorrect brick count of %d for %s %d",
+                                  count, vol_type, volinfo->dist_leaf_count);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        brick_list = GF_MALLOC (120000 * sizeof(*brick_list),gf_common_mt_char);
+
+        if (!brick_list) {
+                ret = -1;
+                goto out;
+        }
+
+        strcpy (brick_list, " ");
+
+        if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+            (volinfo->subvol_count > 1)) {
+                ret = subvol_matcher_init (&subvols, volinfo->subvol_count);
+                if (ret)
+                        goto out;
+        }
+
+        while ( i <= count) {
+                snprintf (key, sizeof (key), "brick%d", i);
+                ret = dict_get_str (dict, key, &brick);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str), "Unable to get %s",
+                                  key);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_DEBUG, "Remove brick count %d brick:"
+                        " %s", i, brick);
+
+                ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo,
+                                                             &brickinfo);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str), "Incorrect brick "
+                                  "%s for volume %s", brick, volname);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+                strcat(brick_list, brick);
+                strcat(brick_list, " ");
+
+                i++;
+                if ((volinfo->type == GF_CLUSTER_TYPE_NONE) ||
+                    (volinfo->brick_count <= volinfo->dist_leaf_count))
+                        continue;
+
+                if (replica_count) {
+                        /* do the validation of bricks here */
+                        /* -2 because i++ is already done, and i starts with 1,
+                           instead of 0 */
+                        diff_count = (volinfo->replica_count - replica_count);
+                        brick_index = (((i -2) / diff_count) * volinfo->replica_count);
+                        tmp_brick_idx = 0;
+                        found = 0;
+                        list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
+                                tmp_brick_idx++;
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "validate brick %s:%s (%d %d %d)",
+                                        tmp->hostname, tmp->path, tmp_brick_idx,
+                                        brick_index, volinfo->replica_count);
+                                if (tmp_brick_idx <= brick_index)
+                                        continue;
+                                if (tmp_brick_idx >
+                                    (brick_index + volinfo->replica_count))
+                                        break;
+                                if ((!strcmp (tmp->hostname,brickinfo->hostname)) &&
+                                    !strcmp (tmp->path, brickinfo->path)) {
+                                        found = 1;
+                                        break;
+                                }
+                        }
+                        if (found)
+                                continue;
+
+                        snprintf (err_str, sizeof (err_str), "Bricks are from "
+                                  "same subvol");
+                        gf_log (this->name, GF_LOG_INFO,
+                                "failed to validate brick %s:%s (%d %d %d)",
+                                tmp->hostname, tmp->path, tmp_brick_idx,
+                                brick_index, volinfo->replica_count);
+                        ret = -1;
+                        /* brick order is not valid */
+                        goto out;
+                }
+
+                /* Find which subvolume the brick belongs to */
+                subvol_matcher_update (subvols, volinfo, brickinfo);
+        }
+
+        /* Check if the bricks belong to the same subvolumes.*/
+        if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+            (volinfo->subvol_count > 1)) {
+                ret = subvol_matcher_verify (subvols, volinfo,
+                                             err_str, sizeof(err_str),
+                                             vol_type);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_REMOVE_BRICK, dict);
+
+out:
+        if (ret) {
+                rsp.op_ret = -1;
+                rsp.op_errno = 0;
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                rsp.op_errstr = err_str;
+                cli_rsp = &rsp;
+                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
+
+                ret = 0; //sent error to cli, prevent second reply
+
+        }
+
+        GF_FREE (brick_list);
+        subvol_matcher_destroy (subvols);
+        free (cli_req.dict.dict_val); //its malloced by xdr
+
+        return ret;
+}
+
+int
+glusterd_handle_remove_brick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_remove_brick);
+}
+
+static int
+_glusterd_restart_gsync_session (dict_t *this, char *key,
+                                 data_t *value, void *data)
+{
+        char                          *slave      = NULL;
+        char                          *slave_buf  = NULL;
+        char                          *path_list  = NULL;
+        char                          *slave_vol  = NULL;
+        char                          *slave_ip   = NULL;
+        char                          *conf_path  = NULL;
+        char                         **errmsg     = NULL;
+        int                            ret        = -1;
+        glusterd_gsync_status_temp_t  *param      = NULL;
+        gf_boolean_t                   is_running = _gf_false;
+
+        param = (glusterd_gsync_status_temp_t *)data;
+
+        GF_ASSERT (param);
+        GF_ASSERT (param->volinfo);
+
+        slave = strchr(value->data, ':');
+        if (slave) {
+                slave++;
+                slave_buf = gf_strdup (slave);
+                if (!slave_buf) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Failed to gf_strdup");
+                        ret = -1;
+                        goto out;
+                }
+        }
+        else
+                return 0;
+
+        ret = dict_set_dynstr (param->rsp_dict, "slave", slave_buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store slave");
+                if (slave_buf)
+                        GF_FREE(slave_buf);
+                goto out;
+        }
+
+        ret = glusterd_get_slave_details_confpath (param->volinfo,
+                                                   param->rsp_dict,
+                                                   &slave_ip, &slave_vol,
+                                                   &conf_path, errmsg);
+        if (ret) {
+                if (*errmsg)
+                        gf_log ("", GF_LOG_ERROR, "%s", *errmsg);
+                else
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to fetch slave or confpath details.");
+                goto out;
+        }
+
+        /* In cases that gsyncd is not running, we will not invoke it
+         * because of add-brick. */
+        ret = glusterd_check_gsync_running_local (param->volinfo->volname,
+                                                  slave, conf_path,
+                                                  &is_running);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "gsync running validation failed.");
+                goto out;
+        }
+        if (_gf_false == is_running) {
+                gf_log ("", GF_LOG_DEBUG, "gsync session for %s and %s is"
+                        " not running on this node. Hence not restarting.",
+                        param->volinfo->volname, slave);
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_get_local_brickpaths (param->volinfo, &path_list);
+        if (!path_list) {
+                gf_log ("", GF_LOG_DEBUG, "This node not being part of"
+                        " volume should not be running gsyncd. Hence"
+                        " no gsyncd process to restart.");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_check_restart_gsync_session (param->volinfo, slave,
+                                                    param->rsp_dict, path_list,
+                                                    conf_path, 0);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to restart gsync session.");
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
+        return ret;
+}
+
+/* op-sm */
+
+int
+glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
+                                char  *bricks, dict_t *dict)
+{
+        char                         *brick          = NULL;
+        int32_t                       i              = 1;
+        char                         *brick_list     = NULL;
+        char                         *free_ptr1      = NULL;
+        char                         *free_ptr2      = NULL;
+        char                         *saveptr        = NULL;
+        int32_t                       ret            = -1;
+        int32_t                       stripe_count   = 0;
+        int32_t                       replica_count  = 0;
+        int32_t                       type           = 0;
+        glusterd_brickinfo_t         *brickinfo      = NULL;
+        glusterd_gsync_status_temp_t  param          = {0, };
+        gf_boolean_t                  restart_needed = 0;
+        char                          msg[1024] __attribute__((unused)) = {0, };
+        int                           caps           = 0;
+
+        GF_ASSERT (volinfo);
+
+        if (bricks) {
+                brick_list = gf_strdup (bricks);
+                free_ptr1 = brick_list;
+        }
+
+        if (count)
+                brick = strtok_r (brick_list+1, " \n", &saveptr);
+
+        if (dict) {
+                ret = dict_get_int32 (dict, "stripe-count", &stripe_count);
+                if (!ret)
+                        gf_log (THIS->name, GF_LOG_INFO,
+                                "stripe-count is set %d", stripe_count);
+
+                ret = dict_get_int32 (dict, "replica-count", &replica_count);
+                if (!ret)
+                        gf_log (THIS->name, GF_LOG_INFO,
+                                "replica-count is set %d", replica_count);
+                ret = dict_get_int32 (dict, "type", &type);
+                if (!ret)
+                        gf_log (THIS->name, GF_LOG_INFO,
+                                "type is set %d, need to change it", type);
+        }
+
+        while ( i <= count) {
+                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret)
+                        goto out;
+                if (stripe_count || replica_count) {
+                        add_brick_at_right_order (brickinfo, volinfo, (i - 1),
+                                                  stripe_count, replica_count);
+                } else {
+                        list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
+                }
+                brick = strtok_r (NULL, " \n", &saveptr);
+                i++;
+                volinfo->brick_count++;
+
+        }
+
+
+        /* Gets changed only if the options are given in add-brick cli */
+        if (type)
+                volinfo->type = type;
+        if (replica_count) {
+                volinfo->replica_count = replica_count;
+        }
+        if (stripe_count) {
+                volinfo->stripe_count = stripe_count;
+        }
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+
+        /* backward compatibility */
+        volinfo->sub_count = ((volinfo->dist_leaf_count == 1) ? 0:
+                              volinfo->dist_leaf_count);
+
+        volinfo->subvol_count = (volinfo->brick_count /
+                                 volinfo->dist_leaf_count);
+
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret)
+                goto out;
+
+        ret = 0;
+        if (GLUSTERD_STATUS_STARTED != volinfo->status)
+                goto out;
+
+        brick_list = gf_strdup (bricks);
+        free_ptr2 = brick_list;
+        i = 1;
+
+        if (count)
+                brick = strtok_r (brick_list+1, " \n", &saveptr);
+#ifdef HAVE_BD_XLATOR
+        if (brickinfo->vg[0])
+                caps = CAPS_BD | CAPS_THIN |
+                        CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+#endif
+
+        while (i <= count) {
+                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                              &brickinfo);
+                if (ret)
+                        goto out;
+#ifdef HAVE_BD_XLATOR
+                /* Check for VG/thin pool if its BD volume */
+                if (brickinfo->vg[0]) {
+                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_CRITICAL, "%s", msg);
+                                goto out;
+                        }
+                        /* if anyone of the brick does not have thin support,
+                           disable it for entire volume */
+                        caps &= brickinfo->caps;
+                } else
+                        caps = 0;
+#endif
+
+                if (uuid_is_null (brickinfo->uuid)) {
+                        ret = glusterd_resolve_brick (brickinfo);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                        brickinfo->hostname, brickinfo->path);
+                                goto out;
+                        }
+                }
+
+                ret = glusterd_brick_start (volinfo, brickinfo,
+                                            _gf_true);
+                if (ret)
+                        goto out;
+                i++;
+                brick = strtok_r (NULL, " \n", &saveptr);
+
+                /* Check if the brick is added in this node, and set
+                 * the restart_needed flag. */
+                if ((!uuid_compare (brickinfo->uuid, MY_UUID)) &&
+                    !restart_needed) {
+                        restart_needed = 1;
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Restart gsyncd session, if it's already "
+                                "running.");
+                }
+        }
+
+        /* If the restart_needed flag is set, restart gsyncd sessions for that
+         * particular master with all the slaves. */
+        if (restart_needed) {
+                param.rsp_dict = dict;
+                param.volinfo = volinfo;
+                dict_foreach (volinfo->gsync_slaves,
+                              _glusterd_restart_gsync_session, &param);
+        }
+        volinfo->caps = caps;
+out:
+        GF_FREE (free_ptr1);
+        GF_FREE (free_ptr2);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+int
+glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
+                                  int force, int *need_migrate)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        int32_t                  ret = -1;
+        glusterd_conf_t         *priv = NULL;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brick);
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                      &brickinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_resolve_brick (brickinfo);
+        if (ret)
+                goto out;
+
+        glusterd_volinfo_reset_defrag_stats (volinfo);
+
+        if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+                /* Only if the brick is in this glusterd, do the rebalance */
+                if (need_migrate)
+                        *need_migrate = 1;
+        }
+
+        if (force) {
+                ret = glusterd_brick_stop (volinfo, brickinfo,
+                                           _gf_true);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Unable to stop "
+                                "glusterfs, ret: %d", ret);
+                }
+                goto out;
+        }
+
+        brickinfo->decommissioned = 1;
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = 0;
+        char                                    *volname = NULL;
+        int                                     count = 0;
+        int                                     i = 0;
+        char                                    *bricks    = NULL;
+        char                                    *brick_list = NULL;
+        char                                    *saveptr = NULL;
+        char                                    *free_ptr = NULL;
+        char                                    *brick = NULL;
+        glusterd_brickinfo_t                    *brickinfo = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        glusterd_conf_t                         *priv = NULL;
+        char                                    msg[2048] = {0,};
+        gf_boolean_t                            brick_alloc = _gf_false;
+        char                                    *all_bricks = NULL;
+        char                                    *str_ret = NULL;
+        gf_boolean_t                            is_force = _gf_false;
+
+        priv = THIS->private;
+        if (!priv)
+                goto out;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to find volume: %s", volname);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_rb_ongoing (volinfo)) {
+                snprintf (msg, sizeof (msg), "Replace brick is in progress on "
+                          "volume %s. Please retry after replace-brick "
+                          "operation is committed or aborted", volname);
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        if (glusterd_is_defrag_on(volinfo)) {
+                snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
+                          "progress. Please retry after completion", volname);
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get count");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "bricks", &bricks);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get bricks");
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        if (bricks) {
+                brick_list = gf_strdup (bricks);
+                all_bricks = gf_strdup (bricks);
+                free_ptr = brick_list;
+        }
+
+        if (count)
+                brick = strtok_r (brick_list+1, " \n", &saveptr);
+
+
+        while ( i < count) {
+                if (!glusterd_store_is_valid_brickpath (volname, brick) ||
+                    !glusterd_is_valid_volfpath (volname, brick)) {
+                        snprintf (msg, sizeof (msg), "brick path %s is "
+                                  "too long", brick);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
+
+                        ret = -1;
+                        goto out;
+
+                }
+
+                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Add-brick: Unable"
+                                " to get brickinfo");
+                        goto out;
+                }
+                brick_alloc = _gf_true;
+
+                ret = glusterd_new_brick_validate (brick, brickinfo, msg,
+                                                   sizeof (msg));
+                if (ret) {
+                        *op_errstr = gf_strdup (msg);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+#ifdef HAVE_BD_XLATOR
+                        if (brickinfo->vg[0]) {
+                                ret = glusterd_is_valid_vg (brickinfo, 1, msg);
+                                if (ret) {
+                                        gf_log (THIS->name, GF_LOG_ERROR, "%s",
+                                                msg);
+                                        *op_errstr = gf_strdup (msg);
+                                        goto out;
+                                }
+                        }
+#endif
+
+                        ret = glusterd_validate_and_create_brickpath (brickinfo,
+                                                          volinfo->volume_id,
+                                                          op_errstr, is_force);
+                        if (ret)
+                                goto out;
+                }
+
+                glusterd_brickinfo_delete (brickinfo);
+                brick_alloc = _gf_false;
+                brickinfo = NULL;
+                brick = strtok_r (NULL, " \n", &saveptr);
+                i++;
+        }
+
+out:
+        GF_FREE (free_ptr);
+        if (brick_alloc && brickinfo)
+                glusterd_brickinfo_delete (brickinfo);
+        GF_FREE (str_ret);
+        GF_FREE (all_bricks);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
+{
+        int                 ret         = -1;
+        char               *volname     = NULL;
+        glusterd_volinfo_t *volinfo     = NULL;
+        char               *errstr      = NULL;
+        int32_t             brick_count = 0;
+        char                msg[2048]   = {0,};
+        int32_t             flag        = 0;
+        gf1_op_commands     cmd         = GF_OP_CMD_NONE;
+        char               *task_id_str = NULL;
+        xlator_t           *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Volume %s does not exist", volname);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_rb_ongoing (volinfo)) {
+                snprintf (msg, sizeof (msg), "Replace brick is in progress on "
+                          "volume %s. Please retry after replace-brick "
+                          "operation is committed or aborted", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "command", &flag);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count");
+                goto out;
+        }
+        cmd = flag;
+
+        ret = -1;
+        switch (cmd) {
+        case GF_OP_CMD_NONE:
+                errstr = gf_strdup ("no remove-brick command issued");
+                goto out;
+
+        case GF_OP_CMD_STATUS:
+                ret = 0;
+                goto out;
+
+        case GF_OP_CMD_START:
+        {
+                if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+                    dict_get (dict, "replica-count")) {
+                        snprintf (msg, sizeof(msg), "Migration of data is not "
+                                  "needed when reducing replica count. Use the"
+                                  " 'force' option");
+                        errstr = gf_strdup (msg);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+
+                if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+                        snprintf (msg, sizeof (msg), "Volume %s needs to be "
+                                  "started before remove-brick (you can use "
+                                  "'force' or 'commit' to override this "
+                                  "behavior)", volinfo->volname);
+                        errstr = gf_strdup (msg);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+                if (!gd_is_remove_brick_committed (volinfo)) {
+                        snprintf (msg, sizeof (msg), "An earlier remove-brick "
+                                  "task exists for volume %s. Either commit it"
+                                  " or stop it before starting a new task.",
+                                  volinfo->volname);
+                        errstr = gf_strdup (msg);
+                        gf_log (this->name, GF_LOG_ERROR, "Earlier remove-brick"
+                                " task exists for volume %s.",
+                                volinfo->volname);
+                        goto out;
+                }
+                if (glusterd_is_defrag_on(volinfo)) {
+                        errstr = gf_strdup("Rebalance is in progress. Please "
+                                           "retry after completion");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        goto out;
+                }
+
+                if (is_origin_glusterd (dict)) {
+                        ret = glusterd_generate_and_set_task_id
+                                (dict, GF_REMOVE_BRICK_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to generate task-id");
+                                goto out;
+                        }
+                } else {
+                        ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Missing remove-brick-id");
+                                ret = 0;
+                        }
+                }
+                break;
+        }
+
+        case GF_OP_CMD_STOP:
+                ret = 0;
+                break;
+
+        case GF_OP_CMD_COMMIT:
+                if (volinfo->decommission_in_progress) {
+                        errstr = gf_strdup ("use 'force' option as migration "
+                                            "is in progress");
+                        goto out;
+                }
+                break;
+
+        case GF_OP_CMD_COMMIT_FORCE:
+                break;
+        }
+
+        ret = dict_get_int32 (dict, "count", &brick_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count");
+                goto out;
+        }
+
+        ret = 0;
+        if (volinfo->brick_count == brick_count) {
+                errstr = gf_strdup ("Deleting all the bricks of the "
+                                    "volume is not allowed");
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        if (ret && errstr) {
+                if (op_errstr)
+                        *op_errstr = errstr;
+        }
+
+        return ret;
+}
+
+int
+glusterd_remove_brick_migrate_cbk (glusterd_volinfo_t *volinfo,
+                                   gf_defrag_status_t status)
+{
+        int                   ret = 0;
+
+#if 0  /* TODO: enable this behavior once cluster-wide awareness comes for
+          defrag cbk function */
+        glusterd_brickinfo_t *brickinfo = NULL;
+        glusterd_brickinfo_t *tmp = NULL;
+
+        switch (status) {
+        case GF_DEFRAG_STATUS_PAUSED:
+        case GF_DEFRAG_STATUS_FAILED:
+                /* No changes required in the volume file.
+                   everything should remain as is */
+                break;
+        case GF_DEFRAG_STATUS_STOPPED:
+                /* Fall back to the old volume file */
+                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks, brick_list) {
+                        if (!brickinfo->decommissioned)
+                                continue;
+                        brickinfo->decommissioned = 0;
+                }
+                break;
+
+        case GF_DEFRAG_STATUS_COMPLETE:
+                /* Done with the task, you can remove the brick from the
+                   volume file */
+                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks, brick_list) {
+                        if (!brickinfo->decommissioned)
+                                continue;
+                        gf_log (THIS->name, GF_LOG_INFO, "removing the brick %s",
+                                brickinfo->path);
+                        brickinfo->decommissioned = 0;
+                        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                            /*TODO: use the 'atomic' flavour of brick_stop*/
+                                ret = glusterd_brick_stop (volinfo, brickinfo);
+                                if (ret) {
+                                        gf_log (THIS->name, GF_LOG_ERROR,
+                                                "Unable to stop glusterfs (%d)", ret);
+                                }
+                        }
+                        glusterd_delete_brick (volinfo, brickinfo);
+                }
+                break;
+
+        default:
+                GF_ASSERT (!"cbk function called with wrong status");
+                break;
+        }
+
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to write volume files (%d)", ret);
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to store volume info (%d)", ret);
+
+
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                ret = glusterd_check_generate_start_nfs ();
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Unable to start nfs process (%d)", ret);
+        }
+
+#endif
+
+        volinfo->decommission_in_progress = 0;
+        return ret;
+}
+
+
+int
+glusterd_op_add_brick (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = 0;
+        char                                    *volname = NULL;
+        glusterd_conf_t                         *priv = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        xlator_t                                *this = NULL;
+        char                                    *bricks = NULL;
+        int32_t                                 count = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get count");
+                goto out;
+        }
+
+
+        ret = dict_get_str (dict, "bricks", &bricks);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get bricks");
+                goto out;
+        }
+
+        ret = glusterd_op_perform_add_bricks (volinfo, count, bricks, dict);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to add bricks");
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+                goto out;
+
+        if (GLUSTERD_STATUS_STARTED == volinfo->status)
+                ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+
+out:
+        return ret;
+}
+
+int
+glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
+{
+        int                     ret            = -1;
+        char                    *volname       = NULL;
+        glusterd_volinfo_t      *volinfo       = NULL;
+        char                    *brick         = NULL;
+        int32_t                 count          = 0;
+        int32_t                 i              = 1;
+        char                    key[256]       = {0,};
+        int32_t                 flag           = 0;
+        char                    err_str[4096]  = {0,};
+        int                     need_rebalance = 0;
+        int                     force          = 0;
+        gf1_op_commands         cmd            = 0;
+        int32_t                 replica_count  = 0;
+        glusterd_brickinfo_t    *brickinfo     = NULL;
+        glusterd_brickinfo_t    *tmp           = NULL;
+        char                    *task_id_str   = NULL;
+        xlator_t                *this          = NULL;
+        dict_t                  *bricks_dict   = NULL;
+        char                    *brick_tmpstr  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "command", &flag);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get command");
+                goto out;
+        }
+        cmd = flag;
+
+        /* Set task-id, if available, in ctx dict for operations other than
+         * start
+         */
+        if (is_origin_glusterd (dict) && (cmd != GF_OP_CMD_START)) {
+                if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                        ret = glusterd_copy_uuid_to_dict
+                                (volinfo->rebal.rebalance_id, dict,
+                                 GF_REMOVE_BRICK_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set remove-brick-id");
+                                goto out;
+                        }
+                }
+        }
+
+        /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
+         * remove-brick */
+        if ((cmd != GF_OP_CMD_START) || (cmd != GF_OP_CMD_STATUS)) {
+                uuid_clear (volinfo->rebal.rebalance_id);
+                volinfo->rebal.op = GD_OP_NONE;
+                dict_unref (volinfo->rebal.dict);
+                volinfo->rebal.dict = NULL;
+        }
+
+        ret = -1;
+        switch (cmd) {
+        case GF_OP_CMD_NONE:
+                goto out;
+
+        case GF_OP_CMD_STATUS:
+                ret = 0;
+                goto out;
+
+        case GF_OP_CMD_STOP:
+        {
+                /* Fall back to the old volume file */
+                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
+                                          brick_list) {
+                        if (!brickinfo->decommissioned)
+                                continue;
+                        brickinfo->decommissioned = 0;
+                }
+                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to create volfiles");
+                        goto out;
+                }
+
+                ret = glusterd_store_volinfo (volinfo,
+                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to store volinfo");
+                        goto out;
+                }
+
+                ret = 0;
+                goto out;
+        }
+
+        case GF_OP_CMD_START:
+                ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Missing remove-brick-id");
+                        ret = 0;
+                } else {
+                        uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
+                        volinfo->rebal.op = GD_OP_REMOVE_BRICK;
+                }
+                force = 0;
+                break;
+
+        case GF_OP_CMD_COMMIT:
+                force = 1;
+                break;
+
+        case GF_OP_CMD_COMMIT_FORCE:
+
+                if (volinfo->decommission_in_progress) {
+                        if (volinfo->rebal.defrag) {
+                                LOCK (&volinfo->rebal.defrag->lock);
+                                /* Fake 'rebalance-complete' so the graph change
+                                   happens right away */
+                                volinfo->rebal.defrag_status =
+                                                GF_DEFRAG_STATUS_COMPLETE;
+
+                                UNLOCK (&volinfo->rebal.defrag->lock);
+                        }
+                        /* Graph change happens in rebalance _cbk function,
+                           no need to do anything here */
+                        /* TODO: '_cbk' function is not doing anything for now */
+                }
+
+                ret = 0;
+                force = 1;
+                break;
+        }
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get count");
+                goto out;
+        }
+
+        /* Save the list of bricks for later usage. Right now this is required
+         * for displaying the task parameters with task status in volume status.
+         */
+        bricks_dict = dict_new ();
+        if (!bricks_dict) {
+                ret = -1;
+                goto out;
+        }
+        ret = dict_set_int32 (bricks_dict, "count", count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to save remove-brick count");
+                goto out;
+        }
+        while ( i <= count) {
+                snprintf (key, 256, "brick%d", i);
+                ret = dict_get_str (dict, key, &brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to get %s",
+                                key);
+                        goto out;
+                }
+
+                brick_tmpstr = gf_strdup (brick);
+                if (!brick_tmpstr) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to duplicate brick name");
+                        goto out;
+                }
+                ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add brick to dict");
+                        goto out;
+                }
+                brick_tmpstr = NULL;
+
+                ret = glusterd_op_perform_remove_brick (volinfo, brick, force,
+                                                        &need_rebalance);
+                if (ret)
+                        goto out;
+                i++;
+        }
+        ret = dict_get_int32 (dict, "replica-count", &replica_count);
+        if (!ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "changing replica count %d to %d on volume %s",
+                        volinfo->replica_count, replica_count,
+                        volinfo->volname);
+                volinfo->replica_count = replica_count;
+                volinfo->sub_count = replica_count;
+                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+                volinfo->subvol_count = (volinfo->brick_count /
+                                         volinfo->dist_leaf_count);
+
+                if (replica_count == 1) {
+                        if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
+                                volinfo->type = GF_CLUSTER_TYPE_NONE;
+                                /* backward compatibility */
+                                volinfo->sub_count = 0;
+                        } else {
+                                volinfo->type = GF_CLUSTER_TYPE_STRIPE;
+                                /* backward compatibility */
+                                volinfo->sub_count = volinfo->dist_leaf_count;
+                        }
+                }
+        }
+        volinfo->rebal.dict = bricks_dict;
+        bricks_dict = NULL;
+
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to create volfiles");
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to store volinfo");
+                goto out;
+        }
+
+        /* Need to reset the defrag/rebalance status accordingly */
+        switch (volinfo->rebal.defrag_status) {
+        case GF_DEFRAG_STATUS_FAILED:
+        case GF_DEFRAG_STATUS_COMPLETE:
+                volinfo->rebal.defrag_status = 0;
+        default:
+                break;
+        }
+        if (!force && need_rebalance) {
+                /* perform the rebalance operations */
+                ret = glusterd_handle_defrag_start
+                        (volinfo, err_str, sizeof (err_str),
+                         GF_DEFRAG_CMD_START_FORCE,
+                         glusterd_remove_brick_migrate_cbk, GD_OP_REMOVE_BRICK);
+
+                if (!ret)
+                        volinfo->decommission_in_progress = 1;
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to start the rebalance");
+                }
+        } else {
+                if (GLUSTERD_STATUS_STARTED == volinfo->status)
+                        ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+        }
+
+out:
+        if (ret && err_str[0] && op_errstr)
+                *op_errstr = gf_strdup (err_str);
+
+        GF_FREE (brick_tmpstr);
+        if (bricks_dict)
+                dict_unref (bricks_dict);
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
new file mode 100644
index 000000000..5786694bd
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
@@ -0,0 +1,4236 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "common-utils.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "run.h"
+#include "syscall.h"
+
+#include <signal.h>
+
+static int
+dict_get_param (dict_t *dict, char *key, char **param);
+
+static int
+glusterd_get_statefile_name (glusterd_volinfo_t *volinfo, char *slave,
+                             char *conf_path, char **statefile);
+
+static int
+glusterd_get_slave_info (char *slave, char **slave_ip,
+                         char **slave_vol, char **op_errstr);
+
+static int
+glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen);
+
+struct gsync_config_opt_vals_ gsync_confopt_vals[] = {
+        {.op_name        = "change_detector",
+         .no_of_pos_vals = 2,
+         .case_sensitive = _gf_true,
+         .values         = {"xsync", "changelog"},
+        },
+        {.op_name        = "special_sync_mode",
+         .no_of_pos_vals = 2,
+         .case_sensitive = _gf_true,
+         .values         = {"partial", "recover"}
+        },
+        {.op_name        = "log-level",
+         .no_of_pos_vals = 5,
+         .case_sensitive = _gf_false,
+         .values         = {"critical", "error", "warning", "info", "debug"}
+        },
+        {.op_name = NULL,
+        },
+};
+
+static char *gsync_reserved_opts[] = {
+        "gluster-command-dir",
+        "pid-file",
+        "remote-gsyncd"
+        "state-file",
+        "session-owner",
+        "state-socket-unencoded",
+        "socketdir",
+        "ignore-deletes",
+        "local-id",
+        "local-path",
+        "slave-id",
+        NULL
+};
+
+int
+__glusterd_handle_sys_exec (rpcsvc_request_t *req)
+{
+        int32_t                 ret     = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req              cli_req = {{0},};
+        glusterd_op_t           cli_op = GD_OP_SYS_EXEC;
+        glusterd_conf_t         *priv   = NULL;
+        char                    *host_uuid = NULL;
+        char                    err_str[2048] = {0,};
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+               } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        return ret;
+}
+
+int
+__glusterd_handle_copy_file (rpcsvc_request_t *req)
+{
+        int32_t                 ret     = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req              cli_req = {{0},};
+        glusterd_op_t           cli_op = GD_OP_COPY_FILE;
+        glusterd_conf_t         *priv   = NULL;
+        char                    *host_uuid = NULL;
+        char                    err_str[2048] = {0,};
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+               } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        return ret;
+}
+
+int
+__glusterd_handle_gsync_set (rpcsvc_request_t *req)
+{
+        int32_t                 ret     = 0;
+        dict_t                  *dict   = NULL;
+        gf_cli_req              cli_req = {{0},};
+        glusterd_op_t           cli_op = GD_OP_GSYNC_SET;
+        char                    *master = NULL;
+        char                    *slave = NULL;
+        char                    operation[256] = {0,};
+        int                     type = 0;
+        glusterd_conf_t         *priv   = NULL;
+        char                    *host_uuid = NULL;
+        char                    err_str[2048] = {0,};
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (req);
+
+       this = THIS;
+       GF_ASSERT (this);
+       priv = this->private;
+       GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret)
+                        goto out;
+
+        }
+
+        ret = dict_get_str (dict, "master", &master);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_INFO, "master not found, while "
+                        "handling "GEOREP" options");
+                master = "(No Master)";
+        }
+
+        ret = dict_get_str (dict, "slave", &slave);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_INFO, "slave not found, while "
+                        "handling "GEOREP" options");
+                slave = "(No Slave)";
+        }
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Command type not found "
+                          "while handling "GEOREP" options");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        switch (type) {
+        case GF_GSYNC_OPTION_TYPE_CREATE:
+                strncpy (operation, "create", sizeof (operation));
+                cli_op = GD_OP_GSYNC_CREATE;
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_START:
+                strncpy (operation, "start", sizeof (operation));
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_STOP:
+                strncpy (operation, "stop", sizeof (operation));
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+                strncpy (operation, "config", sizeof (operation));
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_STATUS:
+                strncpy (operation, "status", sizeof (operation));
+                break;
+        }
+
+        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        return ret;
+}
+
+int
+glusterd_handle_sys_exec (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_sys_exec);
+}
+
+int
+glusterd_handle_copy_file (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_copy_file);
+}
+
+int
+glusterd_handle_gsync_set (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_gsync_set);
+}
+
+/*****
+ *
+ * glusterd_urltransform* internal API
+ *
+ *****/
+
+static void
+glusterd_urltransform_init (runner_t *runner, const char *transname)
+{
+        runinit (runner);
+        runner_add_arg (runner, GSYNCD_PREFIX"/gsyncd");
+        runner_argprintf (runner, "--%s-url", transname);
+}
+
+static void
+glusterd_urltransform_add (runner_t *runner, const char *url)
+{
+        runner_add_arg (runner, url);
+}
+
+static int
+_glusterd_urltransform_add_iter (dict_t *dict, char *key, data_t *value, void *data)
+{
+        runner_t *runner = (runner_t *)data;
+        char *slave = NULL;
+
+        slave = strchr (value->data, ':');
+        GF_ASSERT (slave);
+        slave++;
+        runner_add_arg (runner, slave);
+
+        return 0;
+}
+
+static void
+glusterd_urltransform_free (char **linearr, unsigned n)
+{
+        int i = 0;
+
+        for (; i < n; i++)
+                GF_FREE (linearr[i]);
+
+        GF_FREE (linearr);
+}
+
+static int
+glusterd_urltransform (runner_t *runner, char ***linearrp)
+{
+        char **linearr = NULL;
+        char *line = NULL;
+        unsigned arr_len = 32;
+        unsigned arr_idx = 0;
+        gf_boolean_t error = _gf_false;
+
+        linearr = GF_CALLOC (arr_len, sizeof (char *), gf_gld_mt_linearr);
+        if (!linearr) {
+                error = _gf_true;
+                goto out;
+        }
+
+        runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
+        if (runner_start (runner) != 0) {
+                gf_log ("", GF_LOG_ERROR, "spawning child failed");
+
+                error = _gf_true;
+                goto out;
+        }
+
+        arr_idx = 0;
+        for (;;) {
+                size_t len;
+                line = GF_MALLOC (1024, gf_gld_mt_linebuf);
+                if (!line) {
+                        error = _gf_true;
+                        goto out;
+                }
+
+                if (fgets (line, 1024, runner_chio (runner, STDOUT_FILENO)) ==
+                    NULL)
+                        break;
+
+                len = strlen (line);
+                if (len == 0 || line[len - 1] != '\n') {
+                        GF_FREE (line);
+                        error = _gf_true;
+                        goto out;
+                }
+                line[len - 1] = '\0';
+
+                if (arr_idx == arr_len) {
+                        void *p = linearr;
+                        arr_len <<= 1;
+                        p = GF_REALLOC (linearr, arr_len);
+                        if (!p) {
+                                GF_FREE (line);
+                                error = _gf_true;
+                                goto out;
+                        }
+                        linearr = p;
+                }
+                linearr[arr_idx] = line;
+
+                arr_idx++;
+        }
+
+ out:
+
+        /* XXX chpid field is not exported by run API
+         * but runner_end() does not abort the invoked
+         * process (ie. it might block in waitpid(2))
+         * so we resort to a manual kill a the private field
+         */
+        if (error && runner->chpid > 0)
+                kill (runner->chpid, SIGKILL);
+
+        if (runner_end (runner) != 0)
+                error = _gf_true;
+
+        if (error) {
+                gf_log ("", GF_LOG_ERROR, "reading data from child failed");
+                glusterd_urltransform_free (linearr, arr_idx);
+                return -1;
+        }
+
+        *linearrp = linearr;
+        return arr_idx;
+}
+
+static int
+glusterd_urltransform_single (const char *url, const char *transname,
+                              char ***linearrp)
+{
+        runner_t runner = {0,};
+
+        glusterd_urltransform_init (&runner, transname);
+        glusterd_urltransform_add (&runner, url);
+        return glusterd_urltransform (&runner, linearrp);
+}
+
+
+struct dictidxmark {
+        unsigned isrch;
+        unsigned ithis;
+        char *ikey;
+};
+
+static int
+_dict_mark_atindex (dict_t *dict, char *key, data_t *value, void *data)
+{
+        struct dictidxmark *dim = data;
+
+        if (dim->isrch == dim->ithis)
+                dim->ikey = key;
+
+        dim->ithis++;
+        return 0;
+}
+
+static char *
+dict_get_by_index (dict_t *dict, unsigned i)
+{
+        struct dictidxmark dim = {0,};
+
+        dim.isrch = i;
+        dict_foreach (dict, _dict_mark_atindex, &dim);
+
+        return dim.ikey;
+}
+
+static int
+glusterd_get_slave (glusterd_volinfo_t *vol, const char *slaveurl, char **slavekey)
+{
+        runner_t runner = {0,};
+        int n = 0;
+        int i = 0;
+        char **linearr = NULL;
+
+        glusterd_urltransform_init (&runner, "canonicalize");
+        dict_foreach (vol->gsync_slaves, _glusterd_urltransform_add_iter, &runner);
+        glusterd_urltransform_add (&runner, slaveurl);
+
+        n = glusterd_urltransform (&runner, &linearr);
+        if (n == -1)
+                return -2;
+
+        for (i = 0; i < n - 1; i++) {
+                if (strcmp (linearr[i], linearr[n - 1]) == 0)
+                        break;
+        }
+        glusterd_urltransform_free (linearr, i);
+
+        if (i < n - 1)
+                *slavekey = dict_get_by_index (vol->gsync_slaves, i);
+        else
+                i = -1;
+
+        return i;
+}
+
+
+static int
+glusterd_query_extutil_generic (char *resbuf, size_t blen, runner_t *runner, void *data,
+                                int (*fcbk)(char *resbuf, size_t blen, FILE *fp, void *data))
+{
+        int                 ret = 0;
+
+        runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
+        if (runner_start (runner) != 0) {
+                gf_log ("", GF_LOG_ERROR, "spawning child failed");
+
+                return -1;
+        }
+
+        ret = fcbk (resbuf, blen, runner_chio (runner, STDOUT_FILENO), data);
+
+        ret |= runner_end (runner);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "reading data from child failed");
+
+        return ret ? -1 : 0;
+}
+
+static int
+_fcbk_singleline(char *resbuf, size_t blen, FILE *fp, void *data)
+{
+        char *ptr = NULL;
+
+        errno = 0;
+        ptr = fgets (resbuf, blen, fp);
+        if (ptr) {
+                size_t len = strlen(resbuf);
+                if (len && resbuf[len-1] == '\n')
+                        resbuf[len-1] = '\0'; //strip off \n
+        }
+
+        return errno ? -1 : 0;
+}
+
+static int
+glusterd_query_extutil (char *resbuf, runner_t *runner)
+{
+        return glusterd_query_extutil_generic (resbuf, PATH_MAX, runner, NULL,
+                                               _fcbk_singleline);
+}
+
+static int
+_fcbk_conftodict (char *resbuf, size_t blen, FILE *fp, void *data)
+{
+        char   *ptr = NULL;
+        dict_t *dict = data;
+        char   *v = NULL;
+
+        for (;;) {
+                errno = 0;
+                ptr = fgets (resbuf, blen, fp);
+                if (!ptr)
+                        break;
+                v = resbuf + strlen(resbuf) - 1;
+                while (isspace (*v))
+                        /* strip trailing space */
+                        *v-- = '\0';
+                if (v == resbuf)
+                        /* skip empty line */
+                        continue;
+                v = strchr (resbuf, ':');
+                if (!v)
+                        return -1;
+                *v++ = '\0';
+                while (isspace (*v))
+                        v++;
+                v = gf_strdup (v);
+                if (!v)
+                        return -1;
+                if (dict_set_dynstr (dict, resbuf, v) != 0) {
+                        GF_FREE (v);
+                        return -1;
+                }
+        }
+
+        return errno ? -1 : 0;
+}
+
+static int
+glusterd_gsync_get_config (char *master, char *slave, char *conf_path, dict_t *dict)
+{
+        /* key + value, where value must be able to accommodate a path */
+        char resbuf[256 + PATH_MAX] = {0,};
+        runner_t             runner = {0,};
+
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, "--config-get-all", NULL);
+
+        return glusterd_query_extutil_generic (resbuf, sizeof (resbuf),
+                                               &runner, dict, _fcbk_conftodict);
+}
+
+static int
+glusterd_gsync_get_param_file (char *prmfile, const char *param, char *master,
+                               char *slave, char *conf_path)
+{
+        runner_t            runner = {0,};
+
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, "--config-get", NULL);
+        runner_argprintf (&runner, "%s-file", param);
+
+        return glusterd_query_extutil (prmfile, &runner);
+}
+
+static int
+gsyncd_getpidfile (char *master, char *slave, char *pidfile, char *conf_path)
+{
+        int                ret             = -1;
+        glusterd_conf_t    *priv  = NULL;
+        char               *confpath = NULL;
+        char                conf_buf[PATH_MAX] = "";
+        struct stat         stbuf = {0,};
+
+
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+
+        priv = THIS->private;
+
+        GF_VALIDATE_OR_GOTO ("gsync", master, out);
+        GF_VALIDATE_OR_GOTO ("gsync", slave, out);
+
+        ret = lstat (conf_path, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "Using passed config template(%s).",
+                        conf_path);
+                confpath = conf_path;
+        } else {
+                ret = snprintf (conf_buf, sizeof(conf_buf) - 1,
+                                "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
+                conf_buf[ret] = '\0';
+                confpath = conf_buf;
+                gf_log ("", GF_LOG_DEBUG, "Using default config template(%s).",
+                        confpath);
+        }
+
+        ret = glusterd_gsync_get_param_file (pidfile, "pid", master,
+                                              slave, confpath);
+        if (ret == -1) {
+                ret = -2;
+                gf_log ("", GF_LOG_WARNING, "failed to create the pidfile string");
+                goto out;
+        }
+
+        ret = open (pidfile, O_RDWR);
+
+ out:
+        return ret;
+}
+
+static int
+gsync_status_byfd (int fd)
+{
+        GF_ASSERT (fd >= -1);
+
+        if (lockf (fd, F_TEST, 0) == -1 &&
+            (errno == EAGAIN || errno == EACCES))
+                /* gsyncd keeps the pidfile locked */
+                return 0;
+
+        return -1;
+}
+
+/* status: return 0 when gsync is running
+ * return -1 when not running
+ */
+int
+gsync_status (char *master, char *slave, char *conf_path, int *status)
+{
+        char pidfile[PATH_MAX] = {0,};
+        int  fd                = -1;
+
+        fd = gsyncd_getpidfile (master, slave, pidfile, conf_path);
+        if (fd == -2)
+                return -1;
+
+        *status = gsync_status_byfd (fd);
+
+        sys_close (fd);
+
+        return 0;
+}
+
+
+static int32_t
+glusterd_gsync_volinfo_dict_set (glusterd_volinfo_t *volinfo,
+                                 char *key, char *value)
+{
+        int32_t  ret            = -1;
+        char    *gsync_status   = NULL;
+
+        gsync_status = gf_strdup (value);
+        if (!gsync_status) {
+                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+        ret = dict_set_dynstr (volinfo->dict, key, gsync_status);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to set dict");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return 0;
+}
+
+static int
+glusterd_verify_gsyncd_spawn (char *master, char *slave)
+{
+        int                 ret = 0;
+        runner_t            runner = {0,};
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd",
+                         "--verify", "spawning", NULL);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "spawning child failed");
+                ret = -1;
+                goto out;
+        }
+
+        if (runner_end (&runner) != 0)
+                ret = -1;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+static int
+gsync_verify_config_options (dict_t *dict, char **op_errstr, char *volname)
+{
+        char                         **resopt           = NULL;
+        int                            i                = 0;
+        int                            ret              = -1;
+        char                          *subop            = NULL;
+        char                          *slave            = NULL;
+        char                          *op_name          = NULL;
+        char                          *op_value         = NULL;
+        char                          *t                = NULL;
+        char                           errmsg[PATH_MAX] = "";
+        gf_boolean_t                   banned           = _gf_true;
+        gf_boolean_t                   op_match         = _gf_true;
+        gf_boolean_t                   val_match        = _gf_true;
+        struct gsync_config_opt_vals_ *conf_vals        = NULL;
+
+        if (dict_get_str (dict, "subop", &subop) != 0) {
+                gf_log ("", GF_LOG_WARNING, "missing subop");
+                *op_errstr = gf_strdup ("Invalid config request");
+                return -1;
+        }
+
+        if (dict_get_str (dict, "slave", &slave) != 0) {
+                gf_log ("", GF_LOG_WARNING, GEOREP" CONFIG: no slave given");
+                *op_errstr = gf_strdup ("Slave required");
+                return -1;
+        }
+
+        if (strcmp (subop, "get-all") == 0)
+                return 0;
+
+        if (dict_get_str (dict, "op_name", &op_name) != 0) {
+                gf_log ("", GF_LOG_WARNING, "option name missing");
+                *op_errstr = gf_strdup ("Option name missing");
+                return -1;
+        }
+
+        if (runcmd (GSYNCD_PREFIX"/gsyncd", "--config-check", op_name, NULL)) {
+                ret = glusterd_verify_gsyncd_spawn (volname, slave);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to spawn gsyncd");
+                        return 0;
+                }
+
+                gf_log ("", GF_LOG_WARNING, "Invalid option %s", op_name);
+                *op_errstr = gf_strdup ("Invalid option");
+
+                return -1;
+        }
+
+        if (strcmp (subop, "get") == 0)
+                return 0;
+
+        t = strtail (subop, "set");
+        if (!t)
+                t = strtail (subop, "del");
+        if (!t || (t[0] && strcmp (t, "-glob") != 0)) {
+                gf_log ("", GF_LOG_WARNING, "unknown subop %s", subop);
+                *op_errstr = gf_strdup ("Invalid config request");
+                return -1;
+        }
+
+        if (strtail (subop, "set") &&
+            dict_get_str (dict, "op_value", &op_value) != 0) {
+                gf_log ("", GF_LOG_WARNING, "missing value for set");
+                *op_errstr = gf_strdup ("missing value");
+        }
+
+        /* match option name against reserved options, modulo -/_
+         * difference
+         */
+        for (resopt = gsync_reserved_opts; *resopt; resopt++) {
+                banned = _gf_true;
+                for (i = 0; (*resopt)[i] && op_name[i]; i++) {
+                        if ((*resopt)[i] == op_name[i] ||
+                            ((*resopt)[i] == '-' && op_name[i] == '_'))
+                                continue;
+                        banned = _gf_false;
+                }
+                if (banned) {
+                        gf_log ("", GF_LOG_WARNING, "Reserved option %s", op_name);
+                        *op_errstr = gf_strdup ("Reserved option");
+
+                        return -1;
+                        break;
+                }
+        }
+
+        /* Check options in gsync_confopt_vals for invalid values */
+        for (conf_vals = gsync_confopt_vals; conf_vals->op_name; conf_vals++) {
+                op_match = _gf_true;
+                for (i = 0; conf_vals->op_name[i] && op_name[i]; i++) {
+                        if (conf_vals->op_name[i] == op_name[i] ||
+                            (conf_vals->op_name[i] == '_' && op_name[i] == '-'))
+                                continue;
+                        op_match = _gf_false;
+                }
+
+                if (op_match) {
+                        val_match = _gf_false;
+                        for (i = 0; i < conf_vals->no_of_pos_vals; i++) {
+                                if(conf_vals->case_sensitive){
+                                        if (!strcmp (conf_vals->values[i], op_value))
+                                                val_match = _gf_true;
+                                } else {
+                                        if (!strcasecmp (conf_vals->values[i], op_value))
+                                                val_match = _gf_true;
+                                }
+                        }
+
+                        if (!val_match) {
+                                ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                                "Invalid values (%s) for"
+                                                " option %s", op_value,
+                                                op_name);
+                                errmsg[ret] = '\0';
+
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                return -1;
+                        }
+                }
+        }
+
+        return 0;
+}
+
+static int
+glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
+                                   char *slave, char *conf_path,
+                                   dict_t *rsp_dict, char *node);
+
+static int
+_get_status_mst_slv (dict_t *this, char *key, data_t *value, void *data)
+{
+        glusterd_gsync_status_temp_t  *param = NULL;
+        char                          *slave = NULL;
+        char                          *slave_buf = NULL;
+        char                          *slave_ip  = NULL;
+        char                          *slave_vol = NULL;
+        char                          *errmsg    = NULL;
+        char                           conf_path[PATH_MAX] = "";
+        int                           ret = -1;
+        glusterd_conf_t              *priv = NULL;
+
+        param = (glusterd_gsync_status_temp_t *)data;
+
+        GF_ASSERT (param);
+        GF_ASSERT (param->volinfo);
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
+        slave = strchr(value->data, ':');
+        if (!slave)
+                return 0;
+        slave++;
+
+        ret = glusterd_get_slave_info (slave, &slave_ip, &slave_vol, &errmsg);
+        if (ret) {
+                if (errmsg)
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch "
+                                "slave details. Error: %s", errmsg);
+                else
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to fetch slave details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (conf_path, sizeof(conf_path) - 1,
+                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                        priv->workdir, param->volinfo->volname,
+                        slave_ip, slave_vol);
+        conf_path[ret] = '\0';
+
+        ret = glusterd_get_gsync_status_mst_slv(param->volinfo,
+                                                slave, conf_path,
+                                                param->rsp_dict,
+                                                param->node);
+out:
+
+        if (slave_buf)
+                GF_FREE(slave_buf);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
+        return ret;
+}
+
+
+static int
+_get_max_gsync_slave_num (dict_t *this, char *key, data_t *value, void *data)
+{
+        int  tmp_slvnum = 0;
+        int *slvnum = (int *)data;
+
+        sscanf (key, "slave%d", &tmp_slvnum);
+        if (tmp_slvnum > *slvnum)
+                *slvnum = tmp_slvnum;
+
+        return 0;
+}
+
+static int
+glusterd_remove_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
+                               char **op_errstr)
+{
+        int   zero_slave_entries = _gf_true;
+        int   ret = 0;
+        char *slavekey = NULL;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (slave);
+
+        do {
+                ret = glusterd_get_slave (volinfo, slave, &slavekey);
+                if (ret < 0 && zero_slave_entries) {
+                        ret++;
+                        goto out;
+                }
+                zero_slave_entries = _gf_false;
+                dict_del (volinfo->gsync_slaves, slavekey);
+        } while (ret >= 0);
+
+        ret = glusterd_store_volinfo (volinfo,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                 *op_errstr = gf_strdup ("Failed to store the Volume"
+                                         "information");
+                goto out;
+        }
+ out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+
+}
+
+static int
+glusterd_gsync_get_uuid (char *slave, glusterd_volinfo_t *vol,
+                         uuid_t uuid)
+{
+        int   ret = 0;
+        char *slavekey = NULL;
+        char *slaveentry = NULL;
+        char *t = NULL;
+
+        GF_ASSERT (vol);
+        GF_ASSERT (slave);
+
+        ret = glusterd_get_slave (vol, slave, &slavekey);
+        if (ret < 0) {
+                /* XXX colliding cases of failure and non-extant
+                 * slave... now just doing this as callers of this
+                 * function can make sense only of -1 and 0 as retvals;
+                 * getting at the proper semanticals will involve
+                 * fixing callers as well.
+                 */
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (vol->gsync_slaves, slavekey, &slaveentry);
+        GF_ASSERT (ret == 0);
+
+        t = strchr (slaveentry, ':');
+        GF_ASSERT (t);
+        *t = '\0';
+        ret = uuid_parse (slaveentry, uuid);
+        *t = ':';
+
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_check_gsync_running_local (char *master, char *slave,
+                                    char *conf_path,
+                                    gf_boolean_t *is_run)
+{
+        int                 ret    = -1;
+        int                 ret_status = 0;
+
+        GF_ASSERT (master);
+        GF_ASSERT (slave);
+        GF_ASSERT (is_run);
+
+        *is_run = _gf_false;
+        ret = gsync_status (master, slave, conf_path, &ret_status);
+        if (ret == 0 && ret_status == 0) {
+                *is_run = _gf_true;
+        } else if (ret == -1) {
+                gf_log ("", GF_LOG_WARNING, GEOREP" validation "
+                        " failed");
+                goto out;
+        }
+        ret = 0;
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int
+glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
+                              char *host_uuid, char **op_errstr,
+                              gf_boolean_t is_force)
+{
+        int    ret = 0;
+        int    maxslv = 0;
+        char **linearr = NULL;
+        char  *value = NULL;
+        char  *slavekey = NULL;
+        char  *slaveentry = NULL;
+        char   key[512] = {0, };
+        char  *t = NULL;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (slave);
+        GF_ASSERT (host_uuid);
+
+        ret = glusterd_get_slave (volinfo, slave, &slavekey);
+        switch (ret) {
+        case -2:
+                ret = -1;
+                goto out;
+        case -1:
+                                break;
+        default:
+                if (!is_force)
+                        GF_ASSERT (ret > 0);
+                ret = dict_get_str (volinfo->gsync_slaves, slavekey, &slaveentry);
+                GF_ASSERT (ret == 0);
+
+                /* same-name + same-uuid slave entries should have been filtered
+                 * out in glusterd_op_verify_gsync_start_options(), so we can
+                 * assert an uuid mismatch
+                 */
+                t = strtail (slaveentry, host_uuid);
+                if (!is_force)
+                        GF_ASSERT (!t || *t != ':');
+
+                if (is_force) {
+                        gf_log ("", GF_LOG_DEBUG, GEOREP" has already been "
+                                "invoked for the %s (master) and %s (slave)."
+                                " Allowing without saving info again due to"
+                                " force command.", volinfo->volname, slave);
+                        ret = 0;
+                        goto out;
+                }
+
+                gf_log ("", GF_LOG_ERROR, GEOREP" has already been invoked for "
+                                          "the %s (master) and %s (slave) "
+                                          "from a different machine",
+                                           volinfo->volname, slave);
+                *op_errstr = gf_strdup (GEOREP" already running in "
+                                        "another machine");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_urltransform_single (slave, "normalize", &linearr);
+        if (ret == -1)
+                goto out;
+
+        ret = gf_asprintf (&value,  "%s:%s", host_uuid, linearr[0]);
+        glusterd_urltransform_free (linearr, 1);
+        if (ret  == -1)
+                goto out;
+
+        dict_foreach (volinfo->gsync_slaves, _get_max_gsync_slave_num, &maxslv);
+        snprintf (key, 512, "slave%d", maxslv + 1);
+        ret = dict_set_dynstr (volinfo->gsync_slaves, key, value);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_volinfo (volinfo,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                *op_errstr = gf_strdup ("Failed to store the Volume "
+                                        "information");
+                goto out;
+        }
+        ret = 0;
+ out:
+        return ret;
+}
+
+static int
+glusterd_op_verify_gsync_start_options (glusterd_volinfo_t *volinfo,
+                                        char *slave, char *conf_path,
+                                        char *statefile, char **op_errstr,
+                                        gf_boolean_t is_force)
+{
+        int                     ret = -1;
+        gf_boolean_t            is_running = _gf_false;
+        char                    msg[2048] = {0};
+        uuid_t                  uuid = {0};
+        glusterd_conf_t        *priv = NULL;
+        xlator_t               *this = NULL;
+        struct stat             stbuf = {0,};
+
+        this = THIS;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (slave);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (conf_path);
+        GF_ASSERT (this && this->private);
+
+        priv  = this->private;
+
+        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+                snprintf (msg, sizeof (msg), "Volume %s needs to be started "
+                          "before "GEOREP" start", volinfo->volname);
+                goto out;
+        }
+
+        ret = lstat (statefile, &stbuf);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Session between %s and %s has"
+                          " not been created. Please create session and retry.",
+                          volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        /* Check if the gsync slave info is stored. If not
+         * session has not been created */
+        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Session between %s and %s has"
+                          " not been created. Please create session and retry.",
+                          volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        if (is_force) {
+                ret = 0;
+                goto out;
+        }
+
+        /*Check if the gsync is already started in cmd. inited host
+         * If so initiate add it into the glusterd's priv*/
+        ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                  slave, conf_path,
+                                                  &is_running);
+        if (ret) {
+                snprintf (msg, sizeof (msg), GEOREP" start option "
+                          "validation failed ");
+                goto out;
+        }
+        if (_gf_true == is_running) {
+                snprintf (msg, sizeof (msg), GEOREP " session between"
+                          " %s & %s already started", volinfo->volname,
+                          slave);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Unable to spawn gsyncd");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+        }
+out:
+        if (ret && (msg[0] != '\0')) {
+                *op_errstr = gf_strdup (msg);
+        }
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag)
+{
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (flag);
+
+        if (volinfo->gsync_slaves->count)
+                *flag = _gf_true;
+        else
+                *flag = _gf_false;
+
+        return 0;
+}
+
+static int
+glusterd_op_verify_gsync_running (glusterd_volinfo_t *volinfo,
+                                  char *slave, char *conf_path,
+                                  char **op_errstr)
+{
+        int        pfd               = -1;
+        int        ret               = -1;
+        char       msg[2048]         = {0};
+        char       pidfile[PATH_MAX] = {0,};
+
+        GF_ASSERT (THIS && THIS->private);
+        GF_ASSERT (volinfo);
+        GF_ASSERT (slave);
+        GF_ASSERT (op_errstr);
+
+        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+                snprintf (msg, sizeof (msg), "Volume %s needs to be started "
+                          "before "GEOREP" start", volinfo->volname);
+
+                goto out;
+        }
+
+        pfd = gsyncd_getpidfile (volinfo->volname, slave, pidfile, conf_path);
+        if (pfd == -2) {
+                gf_log ("", GF_LOG_ERROR, GEOREP" stop validation "
+                        "failed for %s & %s", volinfo->volname, slave);
+                ret = -1;
+                goto out;
+        }
+        if (gsync_status_byfd (pfd) == -1) {
+                snprintf (msg, sizeof (msg), GEOREP" session b/w %s & %s is not"
+                          " running on this node.", volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                ret = -1;
+                /* monitor gsyncd already dead */
+                goto out;
+        }
+
+        if (pfd < 0)
+                goto out;
+
+        ret = 0;
+out:
+        if (ret && (msg[0] != '\0')) {
+                *op_errstr = gf_strdup (msg);
+        }
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_verify_gsync_status_opts (dict_t *dict, char **op_errstr)
+{
+        char               *slave  = NULL;
+        char               *volname = NULL;
+        char               errmsg[PATH_MAX] = {0, };
+        gf_boolean_t       exists = _gf_false;
+        glusterd_volinfo_t *volinfo = NULL;
+        int                ret = 0;
+        char               *conf_path = NULL;
+        char               *slave_ip = NULL;
+        char               *slave_vol = NULL;
+        glusterd_conf_t    *priv = NULL;
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "master", &volname);
+        if (ret < 0) {
+                ret = 0;
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if ((ret) || (!exists)) {
+                gf_log ("", GF_LOG_WARNING, "volume name does not exist");
+                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
+                          " exist", volname);
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "slave", &slave);
+        if (ret < 0) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_ip,
+                                                   &slave_vol, &conf_path,
+                                                   op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave  or confpath details.");
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+int
+glusterd_op_gsync_args_get (dict_t *dict, char **op_errstr,
+                            char **master, char **slave, char **host_uuid)
+{
+
+        int             ret = -1;
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        if (master) {
+                ret = dict_get_str (dict, "master", master);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_WARNING, "master not found");
+                        *op_errstr = gf_strdup ("master not found");
+                        goto out;
+                }
+        }
+
+        if (slave) {
+                ret = dict_get_str (dict, "slave", slave);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_WARNING, "slave not found");
+                        *op_errstr = gf_strdup ("slave not found");
+                        goto out;
+                }
+        }
+
+        if (host_uuid) {
+                ret = dict_get_str (dict, "host-uuid", host_uuid);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_WARNING, "host_uuid not found");
+                        *op_errstr = gf_strdup ("host_uuid not found");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_sys_exec (dict_t *dict, char **op_errstr)
+{
+        char             errmsg[PATH_MAX]          = "";
+        char            *command                   = NULL;
+        char             command_path[PATH_MAX]    = "";
+        struct stat      st                        = {0,};
+        int              ret                       = -1;
+        glusterd_conf_t *conf                      = NULL;
+        xlator_t        *this                      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (conf->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "command", &command);
+        if (ret) {
+                strcpy (errmsg, "internal error");
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get command from dict");
+                goto out;
+        }
+
+        /* enforce local occurrence of the command */
+        if (strchr (command, '/')) {
+                strcpy (errmsg, "invalid command name");
+                ret = -1;
+                goto out;
+        }
+
+        sprintf (command_path, GSYNCD_PREFIX"/peer_%s", command);
+        /* check if it's executable */
+        ret = access (command_path, X_OK);
+        if (!ret)
+                /* check if it's a regular file */
+                ret = stat (command_path, &st);
+        if (!ret && !S_ISREG (st.st_mode))
+                ret = -1;
+
+out:
+        if (ret) {
+                if (errmsg[0] == '\0')
+                        snprintf (errmsg, sizeof (errmsg), "%s not found.",
+                                  command);
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_copy_file (dict_t *dict, char **op_errstr)
+{
+        char             abs_filename[PATH_MAX] = "";
+        char             errmsg[PATH_MAX]       = "";
+        char            *filename               = NULL;
+        char            *host_uuid              = NULL;
+        char             uuid_str [64]          = {0};
+        int              ret                    = -1;
+        glusterd_conf_t *priv                   = NULL;
+        struct stat      stbuf                  = {0,};
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
+
+        if (priv->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "host-uuid", &host_uuid);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch"
+                        " host-uuid from dict.");
+                goto out;
+        }
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = dict_get_str (dict, "source", &filename);
+                if (ret < 0) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch"
+                                " filename from dict.");
+                        *op_errstr = gf_strdup ("command unsuccessful");
+                        goto out;
+                }
+                snprintf (abs_filename, sizeof(abs_filename),
+                          "%s/%s", priv->workdir, filename);
+
+                ret = lstat (abs_filename, &stbuf);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Source file"
+                                  " does not exist in %s", priv->workdir);
+                        *op_errstr = gf_strdup (errmsg);
+                        goto out;
+                }
+
+                if (!S_ISREG(stbuf.st_mode)) {
+                        snprintf (errmsg, sizeof (errmsg), "Source file"
+                                 " is not a regular file.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_get_statefile_name (glusterd_volinfo_t *volinfo, char *slave,
+                             char *conf_path, char **statefile)
+{
+        glusterd_conf_t *priv = NULL;
+        int              ret = -1;
+        char            *master    = NULL;
+        char            *buf       = NULL;
+        dict_t          *confd = NULL;
+        char            *confpath = NULL;
+        char             conf_buf[PATH_MAX] = "";
+        struct stat      stbuf = {0,};
+
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+        GF_ASSERT (volinfo);
+
+        master = volinfo->volname;
+
+        confd = dict_new ();
+        if (!confd) {
+                gf_log ("", GF_LOG_ERROR, "Unable to create new dict");
+                goto out;
+        }
+
+        priv = THIS->private;
+
+        ret = lstat (conf_path, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_INFO, "Using passed config template(%s).",
+                        conf_path);
+                confpath = conf_path;
+        } else {
+                ret = snprintf (conf_buf, sizeof(conf_buf) - 1,
+                                "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
+                conf_buf[ret] = '\0';
+                confpath = conf_buf;
+                gf_log ("", GF_LOG_INFO, "Using default config template(%s).",
+                        confpath);
+        }
+
+        ret = glusterd_gsync_get_config (master, slave, confpath,
+                                         confd);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data"
+                        "for %s(master), %s(slave)", master, slave);
+                goto out;
+
+        }
+
+        ret = dict_get_param (confd, "state_file", &buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get state_file's name.");
+                goto out;
+        }
+
+        *statefile = gf_strdup(buf);
+        if (!*statefile) {
+                gf_log ("", GF_LOG_ERROR, "Unable to gf_strdup.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+ out:
+        if (confd)
+                dict_destroy (confd);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
+        return ret;
+}
+
+static int
+glusterd_create_status_file (char *master, char *slave, char *slave_ip,
+                             char *slave_vol, char *status)
+{
+        int                ret    = -1;
+        runner_t           runner = {0,};
+        glusterd_conf_t   *priv   = NULL;
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
+        if (!status) {
+                gf_log ("", GF_LOG_ERROR, "Status Empty");
+                goto out;
+        }
+        gf_log ("", GF_LOG_DEBUG, "slave = %s", slave);
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--create",
+                         status, "-c", NULL);
+        runner_argprintf (&runner, "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                          priv->workdir, master, slave_ip, slave_vol);
+        runner_argprintf (&runner, ":%s", master);
+        runner_add_args  (&runner, slave, NULL);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Creating status file failed.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_verify_slave (char *volname, char *slave_ip, char *slave,
+                       char **op_errstr, gf_boolean_t *is_force_blocker)
+{
+        int32_t          ret                     = -1;
+        runner_t         runner                  = {0,};
+        char             log_file_path[PATH_MAX] = "";
+        char             buf[PATH_MAX]           = "";
+        char            *tmp                     = NULL;
+        char            *save_ptr                = NULL;
+        glusterd_conf_t *priv                    = NULL;
+
+        GF_ASSERT (volname);
+        GF_ASSERT (slave_ip);
+        GF_ASSERT (slave);
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                goto out;
+        }
+
+        snprintf (log_file_path, sizeof(log_file_path),
+                  DEFAULT_LOG_FILE_DIRECTORY"/create_verify_log");
+
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gverify.sh", NULL);
+        runner_argprintf (&runner, "%s", volname);
+        runner_argprintf (&runner, "%s", slave_ip);
+        runner_argprintf (&runner, "%s", slave);
+        runner_argprintf (&runner, "%s", log_file_path);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Not a valid slave");
+                ret = glusterd_gsync_read_frm_status (log_file_path,
+                                                      buf, sizeof(buf));
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to read from %s",
+                                log_file_path);
+                        goto out;
+                }
+
+                /* Tokenize the error message from gverify.sh to figure out
+                 * if the error is a force blocker or not. */
+                tmp = strtok_r (buf, "|", &save_ptr);
+                if (!strcmp (tmp, "FORCE_BLOCKER"))
+                        *is_force_blocker = 1;
+                else {
+                        /* No FORCE_BLOCKER flag present so all that is
+                         * present is the error message. */
+                        *is_force_blocker = 0;
+                        if (tmp)
+                                *op_errstr = gf_strdup (tmp);
+                        ret = -1;
+                        goto out;
+                }
+
+                /* Copy rest of the error message to op_errstr */
+                tmp = strtok_r (NULL, "|", &save_ptr);
+                if (tmp)
+                        *op_errstr = gf_strdup (tmp);
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        unlink (log_file_path);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mountbroker_check (char **slave_ip, char **op_errstr)
+{
+        int   ret             = -1;
+        char *tmp             = NULL;
+        char *save_ptr        = NULL;
+        char *username        = NULL;
+        char *host            = NULL;
+        char errmsg[PATH_MAX] = "";
+
+        GF_ASSERT (slave_ip);
+        GF_ASSERT (*slave_ip);
+
+        /* Checking if hostname has user specified */
+        host = strstr (*slave_ip, "@");
+        if (!host) {
+                gf_log ("", GF_LOG_DEBUG, "No username provided.");
+                ret = 0;
+                goto out;
+        } else {
+                /* Moving the host past the '@' and checking if the
+                 * actual hostname also has '@' */
+                host++;
+                if (strstr (host, "@")) {
+                        gf_log ("", GF_LOG_DEBUG, "host = %s", host);
+                        ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                        "Invalid Hostname (%s).", host);
+                        errmsg[ret] = '\0';
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        if (op_errstr)
+                                *op_errstr = gf_strdup (errmsg);
+                        goto out;
+                }
+
+                /* Fetching the username and hostname
+                 * and checking if the username is non-root */
+                username = strtok_r (*slave_ip, "@", &save_ptr);
+                tmp = strtok_r (NULL, "@", &save_ptr);
+                if (strcmp (username, "root")) {
+                        ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                        "Non-root username (%s@%s) not allowed.",
+                                        username, tmp);
+                        errmsg[ret] = '\0';
+                        if (op_errstr)
+                                *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR,
+                                "Non-Root username not allowed.");
+                        ret = -1;
+                        goto out;
+                }
+
+                *slave_ip = gf_strdup (tmp);
+                if (!*slave_ip) {
+                        gf_log ("", GF_LOG_ERROR, "Out of memory");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_gsync_create (dict_t *dict, char **op_errstr)
+{
+        char               *down_peerstr              = NULL;
+        char               *slave                     = NULL;
+        char               *volname                   = NULL;
+        char               *host_uuid                 = NULL;
+        char               *statefile                 = NULL;
+        char               *slave_ip                  = NULL;
+        char               *slave_vol                 = NULL;
+        char               *conf_path                 = NULL;
+        char                errmsg[PATH_MAX]          = "";
+        char                common_pem_file[PATH_MAX] = "";
+        char                hook_script[PATH_MAX]     = "";
+        char                uuid_str [64]             = "";
+        int                 ret                       = -1;
+        int                 is_pem_push               = -1;
+        gf_boolean_t        is_force                  = -1;
+        gf_boolean_t        is_force_blocker          = -1;
+        gf_boolean_t        exists                    = _gf_false;
+        glusterd_conf_t    *conf                      = NULL;
+        glusterd_volinfo_t *volinfo                   = NULL;
+        struct stat         stbuf                     = {0,};
+        xlator_t           *this                      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = glusterd_op_gsync_args_get (dict, op_errstr, &volname,
+                                          &slave, &host_uuid);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch arguments");
+                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+                return -1;
+        }
+
+        if (conf->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if ((ret) || (!exists)) {
+                gf_log ("", GF_LOG_WARNING, "volume name does not exist");
+                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
+                          " exist", volname);
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+                return -1;
+        }
+
+        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_ip,
+                                                   &slave_vol, &conf_path,
+                                                   op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave or confpath details.");
+                ret = -1;
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = glusterd_are_vol_all_peers_up (volinfo,
+                                                     &conf->peers,
+                                                     &down_peerstr);
+                if ((ret == _gf_false) && !is_force) {
+                        snprintf (errmsg, sizeof (errmsg), "Peer %s,"
+                                  " which is a part of %s volume, is"
+                                  " down. Please bring up the peer and"
+                                  " retry.", down_peerstr,
+                                  volinfo->volname);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        *op_errstr = gf_strdup (errmsg);
+                        GF_FREE (down_peerstr);
+                        down_peerstr = NULL;
+                        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+                        return -1;
+                } else if (ret == _gf_false) {
+                        gf_log ("", GF_LOG_INFO, "Peer %s,"
+                                " which is a part of %s volume, is"
+                                " down. Force creating geo-rep session."
+                                " On bringing up the peer, re-run"
+                                " \"gluster system:: execute"
+                                " gsec_create\" and \"gluster volume"
+                                " geo-replication %s %s create push-pem"
+                                " force\"", down_peerstr, volinfo->volname,
+                                volinfo->volname, slave);
+                }
+
+                /* Checking if slave host is pingable, has proper passwordless
+                 * ssh login setup, slave volume is created, slave vol is empty,
+                 * and if it has enough memory and bypass in case of force if
+                 * the error is not a force blocker */
+                ret = glusterd_verify_slave (volname, slave_ip, slave_vol,
+                                             op_errstr, &is_force_blocker);
+                if (ret) {
+                        if (is_force && !is_force_blocker) {
+                                gf_log ("", GF_LOG_INFO, "%s is not a valid slave"
+                                        " volume. Error: %s. Force creating geo-rep"
+                                        " session.", slave, *op_errstr);
+                        } else {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "%s is not a valid slave volume. Error: %s",
+                                        slave, *op_errstr);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
+                if (!ret && is_pem_push) {
+                        ret = snprintf (common_pem_file,
+                                        sizeof(common_pem_file) - 1,
+                                        "%s"GLUSTERD_COMMON_PEM_PUB_FILE,
+                                        conf->workdir);
+                        common_pem_file[ret] = '\0';
+
+                        ret = snprintf (hook_script, sizeof(hook_script) - 1,
+                                        "%s"GLUSTERD_CREATE_HOOK_SCRIPT,
+                                        conf->workdir);
+                        hook_script[ret] = '\0';
+
+                        ret = lstat (common_pem_file, &stbuf);
+                        if (ret) {
+                                snprintf (errmsg, sizeof (errmsg), "%s"
+                                          " required for push-pem is"
+                                          " not present. Please run"
+                                          " \"gluster system:: execute"
+                                          " gsec_create\"", common_pem_file);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+
+                        ret = lstat (hook_script, &stbuf);
+                        if (ret) {
+                                snprintf (errmsg, sizeof (errmsg),
+                                          "The hook-script (%s) required "
+                                          "for push-pem is not present. "
+                                          "Please install the hook-script "
+                                          "and retry", hook_script);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+
+                        if (!S_ISREG(stbuf.st_mode)) {
+                                snprintf (errmsg, sizeof (errmsg), "%s"
+                                          " required for push-pem is"
+                                          " not a regular file. Please run"
+                                          " \"gluster system:: execute"
+                                          " gsec_create\"", common_pem_file);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        ret = glusterd_get_statefile_name (volinfo, slave, conf_path, &statefile);
+        if (ret) {
+                if (!strstr(slave, "::"))
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "%s is not a valid slave url.", slave);
+                else
+                        snprintf (errmsg, sizeof (errmsg), "Please check gsync "
+                                  "config file. Unable to get statefile's name");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "statefile", statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store statefile path");
+                goto out;
+        }
+
+        ret = lstat (statefile, &stbuf);
+        if (!ret && !is_force) {
+                snprintf (errmsg, sizeof (errmsg), "Session between %s"
+                          " and %s is already created.",
+                          volinfo->volname, slave);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        } else if (!ret)
+                gf_log ("", GF_LOG_INFO, "Session between %s"
+                        " and %s is already created. Force"
+                        " creating again.", volinfo->volname, slave);
+
+        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to spawn gsyncd.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        if (ret && errmsg[0] != '\0')
+                *op_errstr = gf_strdup (errmsg);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
+{
+        int                     ret     = 0;
+        int                     type    = 0;
+        char                    *volname = NULL;
+        char                    *slave   = NULL;
+        char                    *slave_ip = NULL;
+        char                    *slave_vol = NULL;
+        char                    *down_peerstr = NULL;
+        char                    *statefile    = NULL;
+        char                    *path_list    = NULL;
+        char                    *conf_path    = NULL;
+        gf_boolean_t            exists   = _gf_false;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    errmsg[PATH_MAX] = {0,};
+        dict_t                  *ctx = NULL;
+        gf_boolean_t            is_force = 0;
+        gf_boolean_t            is_force_blocker = -1;
+        gf_boolean_t            is_running = _gf_false;
+        uuid_t                  uuid = {0};
+        char                    uuid_str [64] = {0};
+        char                    *host_uuid = NULL;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *conf = NULL;
+        struct stat             stbuf = {0,};
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_WARNING, "command type not found");
+                *op_errstr = gf_strdup ("command unsuccessful");
+                goto out;
+        }
+
+        if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
+                ret = glusterd_verify_gsync_status_opts (dict, op_errstr);
+                goto out;
+        }
+
+        ret = glusterd_op_gsync_args_get (dict, op_errstr,
+                                          &volname, &slave, &host_uuid);
+        if (ret)
+                goto out;
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+
+        if (conf->op_version < 2) {
+                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
+                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
+                          " support the required op version.");
+               *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if ((ret) || (!exists)) {
+                gf_log ("", GF_LOG_WARNING, "volume name does not exist");
+                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
+                          " exist", volname);
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_ip,
+                                                   &slave_vol, &conf_path,
+                                                   op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave or confpath details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_statefile_name (volinfo, slave, conf_path, &statefile);
+        if (ret) {
+                /* Checking if slave host is pingable, has proper passwordless
+                 * ssh login setup */
+                ret = glusterd_verify_slave (volname, slave_ip, slave_vol,
+                                             op_errstr, &is_force_blocker);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "%s is not a valid slave volume. Error: %s",
+                                slave, *op_errstr);
+                        goto out;
+                }
+
+                if (!strstr(slave, "::"))
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "%s is not a valid slave url.", slave);
+                else
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "Unable to get statefile's name");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "statefile", statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store statefile path");
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        /* Allowing stop force to bypass the statefile check
+         * as this command acts as a fail safe method to stop geo-rep
+         * session. */
+        if ((type == GF_GSYNC_OPTION_TYPE_CONFIG) ||
+            ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force) ||
+            (type == GF_GSYNC_OPTION_TYPE_DELETE)) {
+                ret = lstat (statefile, &stbuf);
+                if (ret) {
+                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
+                                  " session between %s and %s does not exist.",
+                                  volinfo->volname, slave);
+                        gf_log ("", GF_LOG_ERROR, "%s. statefile = %s",
+                                errmsg, statefile);
+                        *op_errstr = gf_strdup (errmsg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Check if all peers that are a part of the volume are up or not */
+        if ((type == GF_GSYNC_OPTION_TYPE_DELETE) ||
+            ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force)) {
+                if (!strcmp (uuid_str, host_uuid)) {
+                        ret = glusterd_are_vol_all_peers_up (volinfo,
+                                                             &conf->peers,
+                                                             &down_peerstr);
+                        if (ret == _gf_false) {
+                                snprintf (errmsg, sizeof (errmsg), "Peer %s,"
+                                          " which is a part of %s volume, is"
+                                          " down. Please bring up the peer and"
+                                          " retry.", down_peerstr,
+                                          volinfo->volname);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                GF_FREE (down_peerstr);
+                                down_peerstr = NULL;
+                                goto out;
+                        }
+                }
+        }
+
+        switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+                /* don't attempt to start gsync if replace-brick is
+                 * in progress */
+                if (glusterd_is_rb_ongoing (volinfo)) {
+                        snprintf (errmsg, sizeof(errmsg), "replace-brick is in"
+                                   " progress, not starting geo-replication");
+                        *op_errstr = gf_strdup (errmsg);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_op_verify_gsync_start_options (volinfo, slave,
+                                                              conf_path, statefile,
+                                                              op_errstr, is_force);
+                if (ret)
+                        goto out;
+                ctx = glusterd_op_get_ctx();
+                if (ctx) {
+                        /* gsyncd does a fuse mount to start
+                         * the geo-rep session */
+                        if (!glusterd_is_fuse_available ()) {
+                                gf_log ("glusterd", GF_LOG_ERROR, "Unable to "
+                                        "open /dev/fuse (%s), geo-replication "
+                                        "start failed", strerror (errno));
+                                snprintf (errmsg, sizeof(errmsg),
+                                          "fuse unvailable");
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_STOP:
+                if (!is_force) {
+                        ret = glusterd_op_verify_gsync_running (volinfo, slave,
+                                                                conf_path,
+                                                                op_errstr);
+                        if (ret) {
+                                ret = glusterd_get_local_brickpaths (volinfo,
+                                                                     &path_list);
+                                if (path_list)
+                                        ret = -1;
+                        }
+                }
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+                ret = gsync_verify_config_options (dict, op_errstr, volname);
+                goto out;
+                break;
+
+        case GF_GSYNC_OPTION_TYPE_DELETE:
+                /* Check if the gsync session is still running
+                 * If so ask the user to stop geo-replication first.*/
+                ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
+                if (ret) {
+                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
+                                  " session between %s and %s does not exist.",
+                                  volinfo->volname, slave);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        *op_errstr = gf_strdup (errmsg);
+                        ret = -1;
+                        goto out;
+                } else {
+                        ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                                  slave, conf_path,
+                                                                  &is_running);
+                        if (_gf_true == is_running) {
+                                snprintf (errmsg, sizeof (errmsg), GEOREP
+                                          " session between %s & %s is "
+                                          "still active. Please stop the "
+                                          "session and retry.",
+                                          volinfo->volname, slave);
+                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                                *op_errstr = gf_strdup (errmsg);
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "Unable to spawn gsyncd");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                }
+
+                break;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+stop_gsync (char *master, char *slave, char **msg,
+            char *conf_path, gf_boolean_t is_force)
+{
+        int32_t         ret     = 0;
+        int             pfd     = -1;
+        pid_t           pid     = 0;
+        char            pidfile[PATH_MAX] = {0,};
+        char            buf [1024] = {0,};
+        int             i       = 0;
+
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+
+        pfd = gsyncd_getpidfile (master, slave, pidfile, conf_path);
+        if (pfd == -2 && !is_force) {
+                gf_log ("", GF_LOG_ERROR, GEOREP" stop validation "
+                        " failed for %s & %s", master, slave);
+                ret = -1;
+                goto out;
+        }
+        if (gsync_status_byfd (pfd) == -1 && !is_force) {
+                gf_log ("", GF_LOG_ERROR, "gsyncd b/w %s & %s is not"
+                        " running", master, slave);
+                /* monitor gsyncd already dead */
+                goto out;
+        }
+
+        if (pfd < 0)
+                goto out;
+
+        ret = read (pfd, buf, 1024);
+        if (ret > 0) {
+                pid = strtol (buf, NULL, 10);
+                ret = kill (-pid, SIGTERM);
+                if (ret) {
+                        gf_log ("", GF_LOG_WARNING,
+                                "failed to kill gsyncd");
+                        goto out;
+                }
+                for (i = 0; i < 20; i++) {
+                        if (gsync_status_byfd (pfd) == -1) {
+                                /* monitor gsyncd is dead but worker may
+                                 * still be alive, give some more time
+                                 * before SIGKILL (hack)
+                                 */
+                                usleep (50000);
+                                break;
+                        }
+                        usleep (50000);
+                }
+                kill (-pid, SIGKILL);
+                unlink (pidfile);
+        }
+        ret = 0;
+
+out:
+        sys_close (pfd);
+
+        if (is_force)
+                ret = 0;
+        return ret;
+}
+
+static int
+glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
+                          char *path_list, dict_t *dict,
+                          dict_t *resp_dict, char **op_errstr)
+{
+        int32_t         ret     = -1;
+        char            *op_name = NULL;
+        char            *op_value = NULL;
+        runner_t        runner    = {0,};
+        glusterd_conf_t *priv   = NULL;
+        char            *subop  = NULL;
+        char            *master = NULL;
+        char            *conf_path = NULL;
+        char            *slave_ip  = NULL;
+        char            *slave_vol = NULL;
+        struct stat      stbuf     = {0, };
+
+        GF_ASSERT (slave);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (dict);
+        GF_ASSERT (resp_dict);
+
+        ret = dict_get_str (dict, "subop", &subop);
+        if (ret != 0)
+                goto out;
+
+        if (strcmp (subop, "get") == 0 || strcmp (subop, "get-all") == 0) {
+                /* deferred to cli */
+                gf_log ("", GF_LOG_DEBUG, "Returning 0");
+                return 0;
+        }
+
+        ret = dict_get_str (dict, "op_name", &op_name);
+        if (ret != 0)
+                goto out;
+
+        if (strtail (subop, "set")) {
+                ret = dict_get_str (dict, "op_value", &op_value);
+                if (ret != 0)
+                        goto out;
+        }
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
+        master = "";
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
+        if (volinfo) {
+                master = volinfo->volname;
+                runner_argprintf (&runner, ":%s", master);
+        }
+        runner_add_arg (&runner, slave);
+        runner_argprintf (&runner, "--config-%s", subop);
+        runner_add_arg (&runner, op_name);
+        if (op_value)
+                runner_add_arg (&runner, op_value);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_WARNING, "gsyncd failed to "
+                        "%s %s option for %s %s peers",
+                        subop, op_name, master, slave);
+
+                gf_asprintf (op_errstr, GEOREP" config-%s failed for %s %s",
+                             subop, master, slave);
+
+                goto out;
+        }
+
+        if (!strcmp (op_name, "state_file")) {
+
+                ret = lstat (op_value, &stbuf);
+                if (ret) {
+                        ret = dict_get_str (dict, "slave_ip", &slave_ip);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to fetch slave IP.");
+                                goto out;
+                        }
+
+                        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to fetch slave volume name.");
+                                goto out;
+                        }
+
+                        ret = glusterd_create_status_file (volinfo->volname, slave,
+                                                           slave_ip, slave_vol,
+                                                           "Switching Status File");
+                        if (ret || lstat (op_value, &stbuf)) {
+                                gf_log ("", GF_LOG_ERROR, "Unable to create %s"
+                                        ". Error : %s", op_value,
+                                        strerror (errno));
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+        gf_asprintf (op_errstr, "config-%s successful", subop);
+
+out:
+        if (!ret && volinfo) {
+                ret = glusterd_check_restart_gsync_session (volinfo, slave,
+                                                            resp_dict, path_list,
+                                                            conf_path, 0);
+                if (ret)
+                        *op_errstr = gf_strdup ("internal error");
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen)
+{
+        int                 ret = 0;
+        int           status_fd = -1;
+
+        GF_ASSERT (path);
+        GF_ASSERT (buf);
+        status_fd = open (path, O_RDONLY);
+        if (status_fd == -1) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read gsyncd status"
+                        " file");
+                return -1;
+        }
+        ret = read (status_fd, buf, blen - 1);
+        if (ret > 0) {
+                size_t len = strnlen (buf, ret);
+                /* Ensure there is a NUL byte and that it's not the first.  */
+                if (len == 0 || len == blen - 1) {
+                        ret = -1;
+                } else {
+                        char *p = buf + len - 1;
+                        while (isspace (*p))
+                                *p-- = '\0';
+                        ret = 0;
+                }
+        } else if (ret < 0)
+                gf_log ("", GF_LOG_ERROR, "Status file of gsyncd is corrupt");
+
+        close (status_fd);
+        return ret;
+}
+
+static int
+glusterd_gsync_fetch_status_extra (char *path, char *buf, size_t blen)
+{
+        char sockpath[PATH_MAX] = {0,};
+        struct sockaddr_un   sa = {0,};
+        size_t                l = 0;
+        int                   s = -1;
+        struct pollfd       pfd = {0,};
+        int                 ret = 0;
+
+        l = strlen (buf);
+        /* seek to end of data in buf */
+        buf += l;
+        blen -= l;
+
+        glusterd_set_socket_filepath (path, sockpath, sizeof (sockpath));
+
+        strncpy(sa.sun_path, sockpath, sizeof(sa.sun_path));
+        if (sa.sun_path[sizeof (sa.sun_path) - 1])
+                return -1;
+        sa.sun_family = AF_UNIX;
+
+        s = socket(AF_UNIX, SOCK_STREAM, 0);
+        if (s == -1)
+                return -1;
+        ret = fcntl (s, F_GETFL);
+        if (ret != -1)
+                ret = fcntl (s, F_SETFL, ret | O_NONBLOCK);
+        if (ret == -1)
+                goto out;
+
+        ret = connect (s, (struct sockaddr *)&sa, sizeof (sa));
+        if (ret == -1)
+                goto out;
+        pfd.fd = s;
+        pfd.events = POLLIN;
+        /* we don't want to hang on gsyncd */
+        if (poll (&pfd, 1, 5000) < 1 ||
+            !(pfd.revents & POLLIN)) {
+                ret = -1;
+                goto out;
+        }
+        ret = read(s, buf, blen);
+        /* we expect a terminating 0 byte */
+        if (ret == 0 || (ret > 0 && buf[ret - 1]))
+                ret = -1;
+        if (ret > 0)
+                ret = 0;
+
+ out:
+        close (s);
+        return ret;
+}
+
+static int
+dict_get_param (dict_t *dict, char *key, char **param)
+{
+        char  *dk = NULL;
+        char   *s = NULL;
+        char    x = '\0';
+        int   ret = 0;
+
+        if (dict_get_str (dict, key, param) == 0)
+                return 0;
+
+        dk = gf_strdup (key);
+        if (!key)
+                return -1;
+
+        s = strpbrk (dk, "-_");
+        if (!s)
+                return -1;
+        x = (*s == '-') ? '_' : '-';
+        *s++ = x;
+        while ((s = strpbrk (s, "-_")))
+                *s++ = x;
+
+        ret = dict_get_str (dict, dk, param);
+
+        GF_FREE (dk);
+        return ret;
+}
+
+static int
+glusterd_read_status_file (glusterd_volinfo_t *volinfo, char *slave,
+                           char *conf_path, dict_t *dict, char *node)
+{
+        glusterd_conf_t *priv = NULL;
+        int              ret = 0;
+        char            *statefile = NULL;
+        char            *master    = NULL;
+        char             buf[1024] = "defunct";
+        char             nds[1024] = {0, };
+        char             mst[1024] = {0, };
+        char             slv[1024] = {0, };
+        char             sts[1024] = {0, };
+        char            *bufp = NULL;
+        dict_t          *confd = NULL;
+        int              gsync_count = 0;
+        int              status = 0;
+        char *dyn_node = NULL;
+        char *path_list = NULL;
+
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+        GF_ASSERT (volinfo);
+
+        master = volinfo->volname;
+
+        confd = dict_new ();
+        if (!dict) {
+                gf_log ("", GF_LOG_ERROR, "Not able to create dict.");
+                return -1;
+        }
+
+        priv = THIS->private;
+
+        ret = glusterd_gsync_get_config (master, slave, conf_path,
+                                         confd);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data"
+                        "for %s(master), %s(slave)", master, slave);
+                goto done;
+
+        }
+
+        ret = dict_get_param (confd, "state_file", &statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get state_file's name "
+                        "for %s(master), %s(slave). Please check gsync "
+                        "config file.", master, slave);
+                goto done;
+        }
+        ret = glusterd_gsync_read_frm_status (statefile, buf, sizeof (buf));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read the status"
+                        "file for %s(master), %s(slave)", master, slave);
+                strncpy (buf, "defunct", sizeof (buf));
+                goto done;
+        }
+
+        ret = gsync_status (master, slave, conf_path, &status);
+        if (ret == 0 && status == -1) {
+                if ((strcmp (buf, "Not Started")) &&
+                    (strcmp (buf, "Stopped")))
+                    strncpy (buf, "defunct", sizeof (buf));
+                goto done;
+        } else if (ret == -1) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get gsync status");
+                goto done;
+        }
+
+        if (strcmp (buf, "Stable") != 0)
+                goto done;
+
+        ret = dict_get_param (confd, "state_socket_unencoded", &statefile);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get state_socket_unencoded"
+                        " filepath. Please check gsync config file.");
+                goto done;
+        }
+        ret = glusterd_gsync_fetch_status_extra (statefile, buf, sizeof (buf));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch extra status"
+                        "for %s(master), %s(slave)", master, slave);
+                /* there is a slight chance that this occurs due to race
+                 * -- in that case, the following options all seem bad:
+                 *
+                 * - suppress irregurlar behavior by just leaving status
+                 *   on "OK"
+                 * - freak out users with a misleading "defunct"
+                 * - overload the meaning of the regular error signal
+                 *   mechanism of gsyncd, that is, when status is "faulty"
+                 *
+                 * -- so we just come up with something new...
+                 */
+                strncpy (buf, "N/A", sizeof (buf));
+                goto done;
+        }
+
+ done:
+        if ((!strcmp (buf, "defunct")) ||
+            (!strcmp (buf, "Not Started")) ||
+            (!strcmp (buf, "Stopped"))) {
+                ret = glusterd_get_local_brickpaths (volinfo, &path_list);
+                if (!path_list) {
+                        gf_log ("", GF_LOG_DEBUG, "This node not being part of"
+                                " volume should not be running gsyncd. Hence"
+                                " shouldn't display status for this node.");
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
+
+        if (ret)
+                gsync_count = 1;
+        else
+                gsync_count++;
+
+        (void) snprintf (nds, sizeof (nds), "node%d", gsync_count);
+        dyn_node = gf_strdup (node);
+        if (!dyn_node)
+                goto out;
+        ret = dict_set_dynstr (dict, nds, dyn_node);
+        if (ret) {
+                GF_FREE (dyn_node);
+                goto out;
+        }
+
+        snprintf (mst, sizeof (mst), "master%d", gsync_count);
+        master = gf_strdup (master);
+        if (!master)
+                goto out;
+        ret = dict_set_dynstr (dict, mst, master);
+        if (ret) {
+                GF_FREE (master);
+                goto out;
+        }
+
+        snprintf (slv, sizeof (slv), "slave%d", gsync_count);
+        slave = gf_strdup (slave);
+        if (!slave)
+                goto out;
+        ret = dict_set_dynstr (dict, slv, slave);
+        if (ret) {
+                GF_FREE (slave);
+                goto out;
+        }
+
+        snprintf (sts, sizeof (slv), "status%d", gsync_count);
+        bufp = gf_strdup (buf);
+        if (!bufp)
+                goto out;
+        ret = dict_set_dynstr (dict, sts, bufp);
+        if (ret) {
+                GF_FREE (bufp);
+                goto out;
+        }
+        ret = dict_set_int32 (dict, "gsync-count", gsync_count);
+        if (ret)
+                goto out;
+
+ out:
+        dict_destroy (confd);
+
+        return 0;
+}
+
+int
+glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
+                                      dict_t *resp_dict, char *path_list,
+                                      char *conf_path, gf_boolean_t is_force)
+{
+
+        int                    ret = 0;
+        glusterd_conf_t        *priv = NULL;
+        char                   *status_msg = NULL;
+        gf_boolean_t           is_running = _gf_false;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (slave);
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+
+        priv = THIS->private;
+
+        ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                  slave, conf_path,
+                                                  &is_running);
+        if (!ret && (_gf_true != is_running))
+                /* gsynd not running, nothing to do */
+                goto out;
+
+        ret = stop_gsync (volinfo->volname, slave, &status_msg,
+                          conf_path, is_force);
+        if (ret == 0 && status_msg)
+                ret = dict_set_str (resp_dict, "gsync-status",
+                                    status_msg);
+        if (ret == 0)
+                ret = glusterd_start_gsync (volinfo, slave, path_list,
+                                            conf_path, uuid_utoa(MY_UUID),
+                                            NULL);
+
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_marker_changelog_create_volfile (glusterd_volinfo_t *volinfo)
+{
+        int32_t          ret     = 0;
+
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to create volfile"
+                        " for setting of marker while '"GEOREP" start'");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+                goto out;
+
+        if (GLUSTERD_STATUS_STARTED == volinfo->status)
+                ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+glusterd_set_gsync_knob (glusterd_volinfo_t *volinfo, char *key, int *vc)
+{
+        int   ret          = -1;
+        int   conf_enabled = _gf_false;
+        char *knob_on      = NULL;
+
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+
+        conf_enabled = glusterd_volinfo_get_boolean (volinfo, key);
+        if (conf_enabled == -1) {
+                gf_log ("", GF_LOG_ERROR,
+                        "failed to get key %s from volinfo", key);
+                goto out;
+        }
+
+        ret = 0;
+        if (conf_enabled == _gf_false) {
+                *vc = 1;
+                knob_on = gf_strdup ("on");
+                if (knob_on == NULL) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_gsync_volinfo_dict_set (volinfo,
+                                                       key, knob_on);
+        }
+
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_set_gsync_confs (glusterd_volinfo_t *volinfo)
+{
+        int ret             = -1;
+        int volfile_changed = 0;
+
+        ret = glusterd_set_gsync_knob (volinfo,
+                                       VKEY_MARKER_XTIME, &volfile_changed);
+        if (ret)
+                goto out;
+
+        /**
+         * enable ignore-pid-check blindly as it could be needed for
+         * cascading setups.
+         */
+        ret = glusterd_set_gsync_knob (volinfo, VKEY_MARKER_XTIME_FORCE,
+                                       &volfile_changed);
+        if (ret)
+                goto out;
+
+        ret = glusterd_set_gsync_knob (volinfo,
+                                       VKEY_CHANGELOG, &volfile_changed);
+        if (ret)
+                goto out;
+
+        if (volfile_changed)
+                ret = glusterd_marker_changelog_create_volfile (volinfo);
+
+ out:
+        return ret;
+}
+
+static int
+glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
+                                   char *slave, char *conf_path,
+                                   dict_t *rsp_dict, char *node)
+{
+        char              *statefile = NULL;
+        uuid_t             uuid = {0, };
+        glusterd_conf_t    *priv = NULL;
+        int                ret = 0;
+        struct stat        stbuf = {0, };
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (slave);
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+
+        priv = THIS->private;
+
+        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
+        if (ret) {
+                gf_log ("", GF_LOG_INFO, "geo-replication status %s %s :"
+                        "session is not active", volinfo->volname, slave);
+
+                ret = glusterd_get_statefile_name (volinfo, slave,
+                                                   conf_path, &statefile);
+                if (ret) {
+                        if (!strstr(slave, "::"))
+                                gf_log ("", GF_LOG_INFO,
+                                        "%s is not a valid slave url.", slave);
+                        else
+                                gf_log ("", GF_LOG_INFO, "Unable to get"
+                                        " statefile's name");
+                        ret = 0;
+                        goto out;
+                }
+
+                ret = lstat (statefile, &stbuf);
+                if (ret) {
+                        gf_log ("", GF_LOG_INFO, "%s statefile not present.",
+                                statefile);
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        ret = glusterd_read_status_file (volinfo, slave, conf_path,
+                                         rsp_dict, node);
+out:
+        if (statefile)
+                GF_FREE (statefile);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+static int
+glusterd_get_gsync_status_mst (glusterd_volinfo_t *volinfo, dict_t *rsp_dict,
+                               char *node)
+{
+        glusterd_gsync_status_temp_t  param = {0, };
+
+        GF_ASSERT (volinfo);
+
+        param.rsp_dict = rsp_dict;
+        param.volinfo = volinfo;
+        param.node = node;
+        dict_foreach (volinfo->gsync_slaves, _get_status_mst_slv, &param);
+
+        return 0;
+}
+
+static int
+glusterd_get_gsync_status_all (dict_t *rsp_dict, char *node)
+{
+
+        int32_t                 ret = 0;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+
+        GF_ASSERT (THIS);
+        priv = THIS->private;
+
+        GF_ASSERT (priv);
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                ret = glusterd_get_gsync_status_mst (volinfo, rsp_dict, node);
+                if (ret)
+                        goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+
+}
+
+static int
+glusterd_get_gsync_status (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        char               *slave  = NULL;
+        char               *volname = NULL;
+        char               *conf_path = NULL;
+        char               errmsg[PATH_MAX] = {0, };
+        gf_boolean_t       exists = _gf_false;
+        glusterd_volinfo_t *volinfo = NULL;
+        int                ret = 0;
+        char my_hostname[256] = {0,};
+
+        ret = gethostname(my_hostname, 256);
+        if (ret) {
+                /* stick to N/A */
+                (void) strcpy (my_hostname, "N/A");
+        }
+
+        ret = dict_get_str (dict, "master", &volname);
+        if (ret < 0){
+                ret = glusterd_get_gsync_status_all (rsp_dict, my_hostname);
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if ((ret) || (!exists)) {
+                gf_log ("", GF_LOG_WARNING, "volume name does not exist");
+                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
+                          " exist", volname);
+                *op_errstr = gf_strdup (errmsg);
+                ret = -1;
+                goto out;
+        }
+
+
+        ret = dict_get_str (dict, "slave", &slave);
+        if (ret < 0) {
+                ret = glusterd_get_gsync_status_mst (volinfo,
+                                                     rsp_dict, my_hostname);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
+        ret = glusterd_get_gsync_status_mst_slv (volinfo, slave, conf_path,
+                                                 rsp_dict, my_hostname);
+
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_gsync_delete (glusterd_volinfo_t *volinfo, char *slave, char *slave_ip,
+                       char *slave_vol, char *path_list, dict_t *dict,
+                       dict_t *resp_dict, char **op_errstr)
+{
+        int32_t         ret     = -1;
+        runner_t        runner    = {0,};
+        glusterd_conf_t *priv   = NULL;
+        char            *master = NULL;
+        char            *gl_workdir = NULL;
+        char             geo_rep_dir[PATH_MAX] = "";
+        char            *conf_path = NULL;
+
+        GF_ASSERT (slave);
+        GF_ASSERT (slave_ip);
+        GF_ASSERT (slave_vol);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (dict);
+        GF_ASSERT (resp_dict);
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
+        gl_workdir = priv->workdir;
+        master = "";
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
+                          "--delete", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
+
+        if (volinfo) {
+                master = volinfo->volname;
+                runner_argprintf (&runner, ":%s", master);
+        }
+        runner_add_arg (&runner, slave);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "gsyncd failed to "
+                        "delete session info for %s and %s peers",
+                        master, slave);
+
+                gf_asprintf (op_errstr, "gsyncd failed to "
+                             "delete session info for %s and %s peers",
+                             master, slave);
+
+                goto out;
+        }
+
+        ret = snprintf (geo_rep_dir, sizeof(geo_rep_dir) - 1,
+                        "%s/"GEOREP"/%s_%s_%s", gl_workdir,
+                        volinfo->volname, slave_ip, slave_vol);
+        geo_rep_dir[ret] = '\0';
+
+        ret = rmdir (geo_rep_dir);
+        if (ret) {
+                if (errno == ENOENT)
+                        gf_log ("", GF_LOG_DEBUG, "Geo Rep Dir(%s) Not Present.",
+                                geo_rep_dir);
+                else {
+                        gf_log ("", GF_LOG_ERROR, "Unable to delete "
+                                "Geo Rep Dir(%s). Error: %s", geo_rep_dir,
+                                strerror (errno));
+                        goto out;
+                }
+        }
+
+        ret = 0;
+
+        gf_asprintf (op_errstr, "delete successful");
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_sys_exec (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        char                   buf[PATH_MAX]          = "";
+        char                   cmd_arg_name[PATH_MAX] = "";
+        char                   output_name[PATH_MAX]  = "";
+        char                   errmsg[PATH_MAX]       = "";
+        char                  *ptr                    = NULL;
+        char                  *bufp                   = NULL;
+        char                  *command                = NULL;
+        char                 **cmd_args               = NULL;
+        int                    ret                    = -1;
+        int                    i                      = -1;
+        int                    cmd_args_count         = 0;
+        int                    output_count           = 0;
+        glusterd_conf_t       *priv                   = NULL;
+        runner_t               runner                 = {0,};
+
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "command", &command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get command from dict");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "cmd_args_count", &cmd_args_count);
+        if (ret)
+                gf_log ("", GF_LOG_INFO, "No cmd_args_count");
+
+        if (cmd_args_count) {
+                cmd_args = GF_CALLOC (cmd_args_count, sizeof (char*),
+                                      gf_common_mt_char);
+                if (!cmd_args) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to calloc. "
+                                "Errno = %s", strerror(errno));
+                        goto out;
+                }
+
+                for (i=1; i <= cmd_args_count; i++) {
+                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
+                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
+                                  "cmd_arg_%d", i);
+                        ret = dict_get_str (dict, cmd_arg_name, &cmd_args[i-1]);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s in dict",
+                                        cmd_arg_name);
+                                goto out;
+                        }
+                }
+        }
+
+        runinit (&runner);
+        runner_argprintf (&runner, GSYNCD_PREFIX"/peer_%s", command);
+        for (i=0; i < cmd_args_count; i++)
+                runner_add_arg (&runner, cmd_args[i]);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to "
+                          "execute command. Error : %s",
+                          strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                synclock_lock (&priv->big_lock);
+                goto out;
+        }
+
+        ptr = fgets(buf, sizeof(buf), runner_chio (&runner, STDOUT_FILENO));
+        if (ptr) {
+                ret = dict_get_int32 (rsp_dict, "output_count", &output_count);
+                if (ret)
+                        output_count = 1;
+                else
+                        output_count++;
+                memset (output_name, '\0', sizeof (output_name));
+                snprintf (output_name, sizeof (output_name),
+                          "output_%d", output_count);
+                if (buf[strlen(buf) - 1] == '\n')
+                        buf[strlen(buf) - 1] = '\0';
+                bufp = gf_strdup (buf);
+                if (!bufp)
+                        gf_log ("", GF_LOG_ERROR, "gf_strdup failed.");
+                ret = dict_set_dynstr (rsp_dict, output_name, bufp);
+                if (ret) {
+                        GF_FREE (bufp);
+                        gf_log ("", GF_LOG_ERROR, "output set failed.");
+                }
+                ret = dict_set_int32 (rsp_dict, "output_count", output_count);
+                if (ret)
+                        gf_log ("", GF_LOG_ERROR, "output_count set failed.");
+        }
+
+        ret = runner_end (&runner);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to "
+                          "end. Error : %s",
+                          strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                synclock_lock (&priv->big_lock);
+                goto out;
+        }
+        synclock_lock (&priv->big_lock);
+
+        ret = 0;
+out:
+        if (cmd_args) {
+                GF_FREE (cmd_args);
+                cmd_args = NULL;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_copy_file (dict_t *dict, char **op_errstr)
+{
+        char             abs_filename[PATH_MAX] = "";
+        char             errmsg[PATH_MAX]       = "";
+        char            *filename               = NULL;
+        char            *host_uuid              = NULL;
+        char             uuid_str [64]          = {0};
+        char            *contents               = NULL;
+        char             buf[1024]              = "";
+        int              ret                    = -1;
+        int              fd                     = -1;
+        int              bytes_writen           = 0;
+        int              bytes_read             = 0;
+        int              contents_size          = -1;
+        int              file_mode              = -1;
+        glusterd_conf_t *priv                   = NULL;
+        struct stat      stbuf                  = {0,};
+
+
+        if (THIS)
+                priv = THIS->private;
+        if (priv == NULL) {
+                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+                *op_errstr = gf_strdup ("glusterd defunct");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "host-uuid", &host_uuid);
+        if (ret < 0)
+                goto out;
+
+        ret = dict_get_str (dict, "source", &filename);
+        if (ret < 0) {
+               gf_log ("", GF_LOG_ERROR, "Unable to fetch"
+                       " filename from dict.");
+               *op_errstr = gf_strdup ("command unsuccessful");
+               goto out;
+        }
+        snprintf (abs_filename, sizeof(abs_filename),
+                  "%s/%s", priv->workdir, filename);
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = lstat (abs_filename, &stbuf);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Source file"
+                                 " does not exist in %s", priv->workdir);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+
+                contents = GF_CALLOC(1, stbuf.st_size+1, gf_common_mt_char);
+                if (!contents) {
+                        snprintf (errmsg, sizeof (errmsg),
+                                  "Unable to allocate memory");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+
+                fd = open (abs_filename, O_RDONLY);
+                if (fd < 0) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
+                                  abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+
+                do {
+                        ret = read (fd, buf, sizeof(buf));
+                        if (ret > 0) {
+                                memcpy (contents+bytes_read, buf, ret);
+                                bytes_read += ret;
+                                memset (buf, '\0', sizeof(buf));
+                        }
+                } while (ret > 0);
+
+                if (bytes_read != stbuf.st_size) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to read all "
+                                  "the data from %s", abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_int32 (dict, "contents_size", stbuf.st_size);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " contents size in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+
+                ret = dict_set_int32 (dict, "file_mode",
+                                      (int32_t)stbuf.st_mode);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " file mode in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+
+                ret = dict_set_bin (dict, "common_pem_contents",
+                                    contents, stbuf.st_size);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " pem contents in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+                close (fd);
+        } else {
+                ret = dict_get_bin (dict, "common_pem_contents",
+                                    (void **) &contents);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
+                                  " pem contents in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "contents_size", &contents_size);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
+                                  " contents size in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "file_mode", &file_mode);
+                if (ret) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
+                                  " file mode in dict.");
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+
+                fd = open (abs_filename, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+                if (fd < 0) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
+                                  abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+
+                bytes_writen = write (fd, contents, contents_size);
+
+                if (bytes_writen != contents_size) {
+                        snprintf (errmsg, sizeof (errmsg), "Failed to write"
+                                  " to %s", abs_filename);
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+
+                fchmod (fd, file_mode);
+                close (fd);
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t             ret     = -1;
+        int32_t             type    = -1;
+        dict_t             *ctx    = NULL;
+        dict_t             *resp_dict = NULL;
+        char               *host_uuid = NULL;
+        char               *slave  = NULL;
+        char               *slave_ip  = NULL;
+        char               *slave_vol = NULL;
+        char               *volname = NULL;
+        char               *path_list = NULL;
+        glusterd_volinfo_t *volinfo = NULL;
+        glusterd_conf_t    *priv = NULL;
+        gf_boolean_t        is_force = _gf_false;
+        char               *status_msg = NULL;
+        gf_boolean_t        is_running = _gf_false;
+        char               *conf_path = NULL;
+
+        GF_ASSERT (THIS);
+        GF_ASSERT (THIS->private);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        priv = THIS->private;
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret < 0)
+                goto out;
+
+        ret = dict_get_str (dict, "host-uuid", &host_uuid);
+        if (ret < 0)
+                goto out;
+
+        ctx = glusterd_op_get_ctx ();
+        resp_dict = ctx ? ctx : rsp_dict;
+        GF_ASSERT (resp_dict);
+
+        if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
+                ret = glusterd_get_gsync_status (dict, op_errstr, resp_dict);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "slave", &slave);
+        if (ret < 0)
+                goto out;
+
+        ret = dict_get_str (dict, "slave_ip", &slave_ip);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave volume name.");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave volume name.");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
+        if (dict_get_str (dict, "master", &volname) == 0) {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        gf_log ("", GF_LOG_WARNING, "Volinfo for %s (master) not found",
+                                volname);
+                        goto out;
+                }
+
+                ret = glusterd_get_local_brickpaths (volinfo, &path_list);
+        }
+
+        if (type == GF_GSYNC_OPTION_TYPE_CONFIG) {
+                ret = glusterd_gsync_configure (volinfo, slave, path_list,
+                                                dict, resp_dict, op_errstr);
+
+                ret = dict_set_str (resp_dict, "conf_path", conf_path);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to store conf_file_path.");
+                        goto out;
+                }
+                goto out;
+        }
+
+        if (type == GF_GSYNC_OPTION_TYPE_DELETE) {
+                ret = glusterd_remove_slave_in_info(volinfo, slave, op_errstr);
+                if (ret && !is_force && path_list)
+                        goto out;
+
+                ret = glusterd_gsync_delete (volinfo, slave, slave_ip,
+                                             slave_vol, path_list, dict,
+                                             resp_dict, op_errstr);
+                goto out;
+        }
+
+        if (!volinfo) {
+                ret = -1;
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        if (type == GF_GSYNC_OPTION_TYPE_START) {
+
+                ret = glusterd_set_gsync_confs (volinfo);
+                if (ret != 0) {
+                        gf_log ("", GF_LOG_WARNING, "marker/changelog start failed");
+                        *op_errstr = gf_strdup ("failed to initialize indexing");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_start_gsync (volinfo, slave, path_list,
+                                            conf_path, host_uuid, op_errstr);
+        }
+
+        if (type == GF_GSYNC_OPTION_TYPE_STOP) {
+                ret = glusterd_check_gsync_running_local (volinfo->volname,
+                                                          slave, conf_path,
+                                                          &is_running);
+                if (!ret && !is_force && path_list &&
+                    (_gf_true != is_running)) {
+                        gf_log ("", GF_LOG_WARNING, GEOREP" is not set up for"
+                                "%s(master) and %s(slave)", volname, slave);
+                        *op_errstr = strdup (GEOREP" is not set up");
+                        goto out;
+                }
+
+                ret = stop_gsync (volname, slave, &status_msg, conf_path, is_force);
+                if (ret == 0 && status_msg)
+                        ret = dict_set_str (resp_dict, "gsync-status",
+                                            status_msg);
+                if (ret != 0 && !is_force && path_list)
+                        *op_errstr = gf_strdup ("internal error");
+
+                if (!ret) {
+                        ret = glusterd_create_status_file (volinfo->volname,
+                                                           slave, slave_ip,
+                                                           slave_vol, "Stopped");
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR, "Unable to update"
+                                        "state_file. Error : %s",
+                                        strerror (errno));
+                        }
+                }
+        }
+
+out:
+        if (path_list) {
+                GF_FREE (path_list);
+                path_list = NULL;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_get_slave_details_confpath (glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char **slave_ip, char **slave_vol,
+                                     char **conf_path, char **op_errstr)
+{
+        int                ret                = -1;
+        char               confpath[PATH_MAX] = "";
+        glusterd_conf_t   *priv               = NULL;
+        char              *slave              = NULL;
+
+        GF_ASSERT (THIS);
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "slave", &slave);
+        if (ret || !slave) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave from dict");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_slave_info (slave, slave_ip, slave_vol, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch slave details.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "slave_ip", *slave_ip);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store slave IP.");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "slave_vol", *slave_vol);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store slave volume name.");
+                goto out;
+        }
+
+        ret = snprintf (confpath, sizeof(confpath) - 1,
+                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
+                        priv->workdir, volinfo->volname,
+                        *slave_ip, *slave_vol);
+        confpath[ret] = '\0';
+        *conf_path = gf_strdup (confpath);
+        if (!(*conf_path)) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to gf_strdup. Error: %s", strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "conf_path", *conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to store conf_path");
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
+        return ret;
+
+}
+
+static int
+glusterd_get_slave_info (char *slave, char **slave_ip,
+                         char **slave_vol, char **op_errstr)
+{
+        char     *tmp              = NULL;
+        char     *save_ptr         = NULL;
+        char    **linearr          = NULL;
+        int32_t   ret              = -1;
+        char      errmsg[PATH_MAX] = "";
+
+        ret = glusterd_urltransform_single (slave, "normalize",
+                                            &linearr);
+        if (ret == -1) {
+                ret = snprintf (errmsg, sizeof(errmsg) - 1,
+                                "Invalid Url: %s", slave);
+                errmsg[ret] = '\0';
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "Failed to normalize url");
+                goto out;
+        }
+
+        tmp = strtok_r (linearr[0], "/", &save_ptr);
+        tmp = strtok_r (NULL, "/", &save_ptr);
+        slave = strtok_r (tmp, ":", &save_ptr);
+        if (slave) {
+                ret = glusterd_mountbroker_check (&slave, op_errstr);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Invalid slave url: %s", *op_errstr);
+                        goto out;
+                }
+
+                *slave_ip = gf_strdup (slave);
+                if (!*slave_ip) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Failed to gf_strdup");
+                        ret = -1;
+                        goto out;
+                }
+                gf_log ("", GF_LOG_DEBUG, "Slave IP : %s", *slave_ip);
+                ret = 0;
+        } else {
+                gf_log ("", GF_LOG_ERROR, "Invalid slave name");
+                goto out;
+        }
+
+        slave = strtok_r (NULL, ":", &save_ptr);
+        if (slave) {
+                *slave_vol = gf_strdup (slave);
+                if (!*slave_vol) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Failed to gf_strdup");
+                        ret = -1;
+                        goto out;
+                }
+                gf_log ("", GF_LOG_DEBUG, "Slave Vol : %s", *slave_vol);
+                ret = 0;
+        } else {
+                gf_log ("", GF_LOG_ERROR, "Invalid slave name");
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static void
+runinit_gsyncd_setrx (runner_t *runner, char *conf_path)
+{
+        runinit (runner);
+        runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
+        runner_argprintf (runner, "%s", conf_path);
+        runner_add_arg (runner, "--config-set-rx");
+}
+
+static int
+glusterd_check_gsync_present (int *valid_state)
+{
+        char                buff[PATH_MAX] = {0, };
+        runner_t            runner = {0,};
+        char               *ptr = NULL;
+        int                 ret = 0;
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                if (errno == ENOENT) {
+                        gf_log ("glusterd", GF_LOG_INFO, GEOREP
+                                 " module not installed in the system");
+                        *valid_state = 0;
+                }
+                else {
+                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP
+                                  " module not working as desired");
+                        *valid_state = -1;
+                }
+                goto out;
+        }
+
+        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
+        if (ptr) {
+                if (!strstr (buff, "gsyncd")) {
+                        ret = -1;
+                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
+                                 "working as desired");
+                        *valid_state = -1;
+                        goto out;
+                }
+        } else {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
+                         "working as desired");
+                *valid_state = -1;
+                goto out;
+        }
+
+        ret = 0;
+ out:
+
+        runner_end (&runner);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int
+create_conf_file (glusterd_conf_t *conf, char *conf_path)
+#define RUN_GSYNCD_CMD do {                                                          \
+        ret = runner_run_reuse (&runner);                                            \
+        if (ret == -1) {                                                             \
+                runner_log (&runner, "glusterd", GF_LOG_ERROR, "command failed");    \
+                runner_end (&runner);                                                \
+                goto out;                                                            \
+        }                                                                            \
+        runner_end (&runner);                                                        \
+} while (0)
+{
+        int ret = 0;
+        runner_t runner = {0,};
+        char georepdir[PATH_MAX] = {0,};
+        int valid_state = 0;
+
+        valid_state = -1;
+        ret = glusterd_check_gsync_present (&valid_state);
+        if (-1 == ret) {
+                ret = valid_state;
+                goto out;
+        }
+
+        ret = snprintf (georepdir, sizeof(georepdir) - 1, "%s/"GEOREP,
+                        conf->workdir);
+        georepdir[ret] = '\0';
+
+        /************
+         * master pre-configuration
+         ************/
+
+        /* remote-gsyncd */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "remote-gsyncd", GSYNCD_PREFIX"/gsyncd", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
+                         ".", "^ssh:", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-command-dir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-params",
+                         "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* ssh-command */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "ssh-command");
+        runner_argprintf (&runner,
+                          "ssh -oPasswordAuthentication=no "
+                           "-oStrictHostKeyChecking=no "
+                           "-i %s/secret.pem", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* pid-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "pid-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.pid", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "state-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.status", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-detail-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "state-detail-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-socket */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg (&runner, "state-socket-unencoded");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.socket", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* socketdir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.log",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "gluster-log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* ignore-deletes */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "ignore-deletes", "true", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* special-sync-mode */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* change-detector == changelog */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_arg(&runner, "working-dir");
+        runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+                         DEFAULT_VAR_RUN_DIRECTORY);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /************
+         * slave pre-configuration
+         ************/
+
+        /* gluster-command-dir */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner, "gluster-params",
+                         "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* MountBroker log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "log-file-mbr",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-log-file */
+        runinit_gsyncd_setrx (&runner, conf_path);
+        runner_add_args (&runner,
+                         "gluster-log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.gluster.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+ out:
+        return ret ? -1 : 0;
+}
+
+static int
+glusterd_create_essential_dir_files (glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char *slave, char *slave_ip,
+                                     char *slave_vol, char **op_errstr)
+{
+        int                ret              = -1;
+        char              *conf_path        = NULL;
+        char              *statefile        = NULL;
+        char               buf[PATH_MAX]    = "";
+        char               errmsg[PATH_MAX] = "";
+        glusterd_conf_t   *conf             = NULL;
+        struct stat        stbuf            = {0,};
+
+        GF_ASSERT (THIS);
+        conf = THIS->private;
+
+        ret = dict_get_str (dict, "conf_path", &conf_path);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch conf file path.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "statefile", &statefile);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch statefile path.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = snprintf (buf, sizeof(buf) - 1, "%s/"GEOREP"/%s_%s_%s",
+                        conf->workdir, volinfo->volname, slave_ip, slave_vol);
+        buf[ret] = '\0';
+        ret = mkdir_p (buf, 0777, _gf_true);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
+                          ". Error : %s", buf, strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = snprintf (buf, PATH_MAX, DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/%s",
+                        volinfo->volname);
+        buf[ret] = '\0';
+        ret = mkdir_p (buf, 0777, _gf_true);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
+                          ". Error : %s", buf, strerror (errno));
+                *op_errstr = gf_strdup (errmsg);
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = lstat (conf_path, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "Session already running."
+                        " Not creating config file again.");
+        } else {
+                ret = create_conf_file (conf, conf_path);
+                if (ret || lstat (conf_path, &stbuf)) {
+                        snprintf (errmsg, sizeof (errmsg), "Failed to create"
+                                  " config file(%s).", conf_path);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        goto out;
+                }
+        }
+
+        ret = lstat (statefile, &stbuf);
+        if (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "Session already running."
+                        " Not creating status file again.");
+                goto out;
+        } else {
+                ret = glusterd_create_status_file (volinfo->volname, slave,
+                                                   slave_ip, slave_vol,
+                                                   "Not Started");
+                if (ret || lstat (statefile, &stbuf)) {
+                        snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
+                                  ". Error : %s", statefile, strerror (errno));
+                        *op_errstr = gf_strdup (errmsg);
+                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_gsync_create (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        char                common_pem_file[PATH_MAX] = "";
+        char                errmsg[PATH_MAX]          = "";
+        char                hooks_args[PATH_MAX]      = "";
+        char                uuid_str [64]             = "";
+        char               *host_uuid                 = NULL;
+        char               *slave_ip                  = NULL;
+        char               *slave_vol                 = NULL;
+        char               *arg_buf                   = NULL;
+        char               *volname                   = NULL;
+        char               *slave                     = NULL;
+        int32_t             ret                       = -1;
+        int32_t             is_pem_push               = -1;
+        gf_boolean_t        is_force                  = -1;
+        glusterd_conf_t    *conf                      = NULL;
+        glusterd_volinfo_t *volinfo                   = NULL;
+
+        GF_ASSERT (THIS);
+        conf = THIS->private;
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        ret = glusterd_op_gsync_args_get (dict, op_errstr,
+                                          &volname, &slave, &host_uuid);
+        if (ret)
+                goto out;
+
+        snprintf (common_pem_file, sizeof(common_pem_file),
+                  "%s"GLUSTERD_COMMON_PEM_PUB_FILE, conf->workdir);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Volinfo for %s"
+                        " (master) not found", volname);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch slave volume name.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "slave_ip", &slave_ip);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg),
+                          "Unable to fetch slave IP.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                ret = -1;
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        if (!strcmp (uuid_str, host_uuid)) {
+                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
+                if (!ret && is_pem_push) {
+                        gf_log ("", GF_LOG_DEBUG, "Trying to setup"
+                                " pem files in slave");
+                        is_pem_push = 1;
+                } else
+                        is_pem_push = 0;
+
+                snprintf(hooks_args, sizeof(hooks_args),
+                         "is_push_pem=%d pub_file=%s slave_ip=%s",
+                         is_pem_push, common_pem_file, slave_ip);
+
+        } else
+                snprintf(hooks_args, sizeof(hooks_args),
+                         "This argument will stop the hooks script");
+
+        arg_buf = gf_strdup (hooks_args);
+        if (!arg_buf) {
+                gf_log ("", GF_LOG_ERROR, "Failed to"
+                        " gf_strdup");
+                if (is_force) {
+                        ret = 0;
+                        goto create_essentials;
+                }
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "hooks_args", arg_buf);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Failed to set"
+                        " hooks_args in dict.");
+                if (is_force) {
+                        ret = 0;
+                        goto create_essentials;
+                }
+                goto out;
+        }
+
+create_essentials:
+
+        ret = glusterd_create_essential_dir_files (volinfo, dict, slave,
+                                                   slave_ip, slave_vol,
+                                                   op_errstr);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_slave_in_info (volinfo, slave,
+                                            host_uuid, op_errstr,
+                                            is_force);
+        if (ret) {
+                snprintf (errmsg, sizeof (errmsg), "Unable to store"
+                          " slave info.");
+                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
new file mode 100644
index 000000000..71d076624
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -0,0 +1,4237 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <inttypes.h>
+
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "compat.h"
+#include "dict.h"
+#include "protocol-common.h"
+#include "xlator.h"
+#include "logging.h"
+#include "timer.h"
+#include "defaults.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "statedump.h"
+#include "run.h"
+#include "glusterd-mem-types.h"
+#include "glusterd.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "glusterd-locks.h"
+
+#include "glusterd1-xdr.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "rpc-clnt.h"
+#include "glusterd-volgen.h"
+#include "glusterd-mountbroker.h"
+
+#include <sys/resource.h>
+#include <inttypes.h>
+
+#include "defaults.c"
+#include "common-utils.h"
+
+#include "globals.h"
+#include "glusterd-syncop.h"
+
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
+extern uuid_t global_txn_id;
+
+int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
+                                rpc_clnt_event_t event,
+                                void *data, rpc_clnt_notify_t notify_fn)
+{
+        glusterd_conf_t *priv = THIS->private;
+        int             ret   = -1;
+        synclock_lock (&priv->big_lock);
+        ret = notify_fn (rpc, mydata, event, data);
+        synclock_unlock (&priv->big_lock);
+        return ret;
+}
+
+int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn)
+{
+        glusterd_conf_t *priv = THIS->private;
+        int             ret   = -1;
+
+        synclock_lock (&priv->big_lock);
+        ret = actor_fn (req);
+        synclock_unlock (&priv->big_lock);
+
+        return ret;
+}
+
+static int
+glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
+                            char *hostname, int port,
+                            gd1_mgmt_friend_req *friend_req)
+{
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_friend_sm_event_t      *event = NULL;
+        glusterd_friend_req_ctx_t       *ctx = NULL;
+        char                            rhost[UNIX_PATH_MAX + 1] = {0};
+        uuid_t                          friend_uuid = {0};
+        dict_t                          *dict = NULL;
+
+        uuid_parse (uuid_utoa (uuid), friend_uuid);
+        if (!port)
+                port = GF_DEFAULT_BASE_PORT;
+
+        ret = glusterd_remote_hostname_get (req, rhost, sizeof (rhost));
+        ret = glusterd_friend_find (uuid, rhost, &peerinfo);
+
+        if (ret) {
+                ret = glusterd_xfer_friend_add_resp (req, hostname, rhost, port,
+                                                     -1, GF_PROBE_UNKNOWN_PEER);
+                if (friend_req->vols.vols_val) {
+                        free (friend_req->vols.vols_val);
+                        friend_req->vols.vols_val = NULL;
+                }
+                goto out;
+        }
+
+        ret = glusterd_friend_sm_new_event
+                        (GD_FRIEND_EVENT_RCVD_FRIEND_REQ, &event);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "event generation failed: %d", ret);
+                return ret;
+        }
+
+        event->peerinfo = peerinfo;
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_friend_req_ctx_t);
+
+        if (!ctx) {
+                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, uuid);
+        if (hostname)
+                ctx->hostname = gf_strdup (hostname);
+        ctx->req = req;
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (friend_req->vols.vols_val,
+                                friend_req->vols.vols_len,
+                                &dict);
+
+        if (ret)
+                goto out;
+        else
+                dict->extra_stdfree = friend_req->vols.vols_val;
+
+        ctx->vols = dict;
+        event->ctx = ctx;
+
+        ret = glusterd_friend_sm_inject_event (event);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject event %d, "
+                        "ret = %d", event->event, ret);
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (0 != ret) {
+                if (ctx && ctx->hostname)
+                        GF_FREE (ctx->hostname);
+                GF_FREE (ctx);
+                if (dict) {
+                        if ((!dict->extra_stdfree) &&
+                            friend_req->vols.vols_val)
+                                free (friend_req->vols.vols_val);
+                        dict_unref (dict);
+                } else {
+                    free (friend_req->vols.vols_val);
+                }
+                GF_FREE (event);
+        } else {
+                if (peerinfo && (0 == peerinfo->connected))
+                        ret = GLUSTERD_CONNECTION_AWAITED;
+        }
+        return ret;
+}
+
+static int
+glusterd_handle_unfriend_req (rpcsvc_request_t *req, uuid_t  uuid,
+                              char *hostname, int port)
+{
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_friend_sm_event_t      *event = NULL;
+        glusterd_friend_req_ctx_t       *ctx = NULL;
+
+        if (!port)
+                port = GF_DEFAULT_BASE_PORT;
+
+        ret = glusterd_friend_find (uuid, hostname, &peerinfo);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Received remove-friend from unknown peer %s",
+                        hostname);
+                ret = glusterd_xfer_friend_remove_resp (req, hostname,
+                                                        port);
+                goto out;
+        }
+
+        ret = glusterd_friend_sm_new_event
+                        (GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND, &event);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "event generation failed: %d", ret);
+                return ret;
+        }
+
+        event->peerinfo = peerinfo;
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_friend_req_ctx_t);
+
+        if (!ctx) {
+                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, uuid);
+        if (hostname)
+                ctx->hostname = gf_strdup (hostname);
+        ctx->req = req;
+
+        event->ctx = ctx;
+
+        ret = glusterd_friend_sm_inject_event (event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject event %d, "
+                        "ret = %d", event->event, ret);
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (0 != ret) {
+                if (ctx && ctx->hostname)
+                        GF_FREE (ctx->hostname);
+                GF_FREE (ctx);
+        }
+
+        return ret;
+}
+
+static int
+glusterd_add_peer_detail_to_dict (glusterd_peerinfo_t   *peerinfo,
+                                  dict_t  *friends, int   count)
+{
+
+        int             ret = -1;
+        char            key[256] = {0, };
+        char           *peer_uuid_str = NULL;
+
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (friends);
+
+        snprintf (key, 256, "friend%d.uuid", count);
+        peer_uuid_str = gd_peer_uuid_str (peerinfo);
+        ret = dict_set_str (friends, key, peer_uuid_str);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "friend%d.hostname", count);
+        ret = dict_set_str (friends, key, peerinfo->hostname);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "friend%d.port", count);
+        ret = dict_set_int32 (friends, key, peerinfo->port);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "friend%d.stateId", count);
+        ret = dict_set_int32 (friends, key, peerinfo->state.state);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "friend%d.state", count);
+        ret = dict_set_str (friends, key,
+                    glusterd_friend_sm_state_name_get(peerinfo->state.state));
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "friend%d.connected", count);
+        ret = dict_set_int32 (friends, key, (int32_t)peerinfo->connected);
+        if (ret)
+                goto out;
+
+out:
+        return ret;
+}
+
+struct args_pack {
+    dict_t *dict;
+    int vol_count;
+    int opt_count;
+};
+
+static int
+_build_option_key (dict_t *d, char *k, data_t *v, void *tmp)
+{
+        char                    reconfig_key[256] = {0, };
+        struct args_pack        *pack             = NULL;
+        int                     ret               = -1;
+
+        pack = tmp;
+        if (strcmp (k, GLUSTERD_GLOBAL_OPT_VERSION) == 0)
+                return 0;
+        snprintf (reconfig_key, 256, "volume%d.option.%s",
+                  pack->vol_count, k);
+        ret = dict_set_str (pack->dict, reconfig_key, v->data);
+        if (0 == ret)
+                pack->opt_count++;
+
+        return 0;
+}
+
+int
+glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
+                                    dict_t  *volumes, int count)
+{
+
+        int                     ret = -1;
+        char                    key[256] = {0, };
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        char                    *buf = NULL;
+        int                     i = 1;
+        dict_t                  *dict = NULL;
+        glusterd_conf_t         *priv = NULL;
+        char                    *volume_id_str  = NULL;
+        struct args_pack        pack = {0,};
+        xlator_t                *this = NULL;
+#ifdef HAVE_BD_XLATOR
+        int                     caps = 0;
+#endif
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (volumes);
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        snprintf (key, 256, "volume%d.name", count);
+        ret = dict_set_str (volumes, key, volinfo->volname);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "volume%d.type", count);
+        ret = dict_set_int32 (volumes, key, volinfo->type);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "volume%d.status", count);
+        ret = dict_set_int32 (volumes, key, volinfo->status);
+        if (ret)
+                goto out;
+
+        /* As of now, the snap volumes are also displayed as part of
+           volume info command. So this change is to display whether
+           the volume is original volume or the snap_volume. If
+           displaying of snap volumes in volume info o/p is not needed
+           this should be removed.
+        */
+        snprintf (key, 256, "volume%d.snap_volume", count);
+        ret = dict_set_int32 (volumes, key, volinfo->is_snap_volume);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to set whether "
+                        "the volume is a snap volume or actual volume (%s)",
+                        volinfo->volname);
+                goto out;
+        }
+
+        snprintf (key, 256, "volume%d.brick_count", count);
+        ret = dict_set_int32 (volumes, key, volinfo->brick_count);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "volume%d.dist_count", count);
+        ret = dict_set_int32 (volumes, key, volinfo->dist_leaf_count);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "volume%d.stripe_count", count);
+        ret = dict_set_int32 (volumes, key, volinfo->stripe_count);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "volume%d.replica_count", count);
+        ret = dict_set_int32 (volumes, key, volinfo->replica_count);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "volume%d.transport", count);
+        ret = dict_set_int32 (volumes, key, volinfo->transport_type);
+        if (ret)
+                goto out;
+
+        volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
+        if (!volume_id_str)
+                goto out;
+
+        snprintf (key, sizeof (key), "volume%d.volume_id", count);
+        ret = dict_set_dynstr (volumes, key, volume_id_str);
+        if (ret)
+                goto out;
+
+        snprintf (key, 256, "volume%d.rebalance", count);
+        ret = dict_set_int32 (volumes, key, volinfo->rebal.defrag_cmd);
+        if (ret)
+                goto out;
+
+#ifdef HAVE_BD_XLATOR
+        if (volinfo->caps) {
+                caps = 0;
+                snprintf (key, 256, "volume%d.xlator0", count);
+                buf = GF_MALLOC (256, gf_common_mt_char);
+                if (!buf) {
+                        ret = ENOMEM;
+                        goto out;
+                }
+                if (volinfo->caps & CAPS_BD)
+                        snprintf (buf, 256, "BD");
+                ret = dict_set_dynstr (volumes, key, buf);
+                if (ret) {
+                        GF_FREE (buf);
+                        goto out;
+                }
+
+                if (volinfo->caps & CAPS_THIN) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "thin");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret) {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+                if (volinfo->caps & CAPS_OFFLOAD_COPY) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "offload_copy");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret) {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+                if (volinfo->caps & CAPS_OFFLOAD_SNAPSHOT) {
+                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
+                                  caps++);
+                        buf = GF_MALLOC (256, gf_common_mt_char);
+                        if (!buf) {
+                                ret = ENOMEM;
+                                goto out;
+                        }
+                        snprintf (buf, 256, "offload_snapshot");
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret)  {
+                                GF_FREE (buf);
+                                goto out;
+                        }
+                }
+
+        }
+#endif
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                char    brick[1024] = {0,};
+                char    brick_uuid[64] = {0,};
+                snprintf (key, 256, "volume%d.brick%d", count, i);
+                snprintf (brick, 1024, "%s:%s", brickinfo->hostname,
+                          brickinfo->path);
+                buf = gf_strdup (brick);
+                ret = dict_set_dynstr (volumes, key, buf);
+                if (ret)
+                        goto out;
+                snprintf (key, 256, "volume%d.brick%d.uuid", count, i);
+                snprintf (brick_uuid, 64, "%s", uuid_utoa (brickinfo->uuid));
+                buf = gf_strdup (brick_uuid);
+                if (!buf)
+                        goto out;
+                ret = dict_set_dynstr (volumes, key, buf);
+                if (ret)
+                        goto out;
+
+#ifdef HAVE_BD_XLATOR
+                if (volinfo->caps & CAPS_BD) {
+                        snprintf (key, 256, "volume%d.vg%d", count, i);
+                        snprintf (brick, 1024, "%s", brickinfo->vg);
+                        buf = gf_strdup (brick);
+                        ret = dict_set_dynstr (volumes, key, buf);
+                        if (ret)
+                                goto out;
+                }
+#endif
+                i++;
+        }
+
+        dict = volinfo->dict;
+        if (!dict) {
+                ret = 0;
+                goto out;
+        }
+
+        pack.dict = volumes;
+        pack.vol_count = count;
+        pack.opt_count = 0;
+        dict_foreach (dict, _build_option_key, (void *) &pack);
+        dict_foreach (priv->opts, _build_option_key, &pack);
+
+        snprintf (key, 256, "volume%d.opt_count", pack.vol_count);
+        ret = dict_set_int32 (volumes, key, pack.opt_count);
+out:
+        return ret;
+}
+
+int
+glusterd_friend_find (uuid_t uuid, char *hostname,
+                      glusterd_peerinfo_t **peerinfo)
+{
+        int     ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (uuid) {
+                ret = glusterd_friend_find_by_uuid (uuid, peerinfo);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                 "Unable to find peer by uuid: %s",
+                                 uuid_utoa (uuid));
+                } else {
+                        goto out;
+                }
+
+        }
+
+        if (hostname) {
+                ret = glusterd_friend_find_by_hostname (hostname, peerinfo);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Unable to find hostname: %s", hostname);
+                } else {
+                        goto out;
+                }
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                       char *err_str, size_t err_len)
+{
+        int32_t                     ret             = -1;
+        dict_t                     *dict            = NULL;
+        xlator_t                   *this            = NULL;
+        glusterd_conf_t            *priv            = NULL;
+        int32_t                     locked          = 0;
+        char                       *tmp             = NULL;
+        char                       *volname         = NULL;
+        uuid_t                     *txn_id          = NULL;
+        uuid_t                     *originator_uuid = NULL;
+        glusterd_op_info_t          txn_op_info     = {{0},};
+        glusterd_op_sm_event_type_t event_type      = GD_OP_EVENT_NONE;
+
+        GF_ASSERT (req);
+        GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
+        GF_ASSERT (NULL != ctx);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        dict = ctx;
+
+        /* Generate a transaction-id for this operation and
+         * save it in the dict. This transaction id distinguishes
+         * each transaction, and helps separate opinfos in the
+         * op state machine. */
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!txn_id)
+                goto out;
+
+        uuid_generate (*txn_id);
+
+        ret = dict_set_bin (dict, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Transaction_id = %s", uuid_utoa (*txn_id));
+
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator uuid.");
+                goto out;
+        }
+
+        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+        if (priv->op_version < 3) {
+                ret = glusterd_lock (MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock on localhost, ret: %d",
+                                ret);
+                        snprintf (err_str, err_len,
+                                  "Another transaction is in progress. "
+                                  "Please try again after sometime.");
+                        goto out;
+                }
+        } else {
+                /* If no volname is given as a part of the command, locks will
+                 * not be held */
+                ret = dict_get_str (dict, "volname", &tmp);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to get volume "
+                                "name");
+                        goto local_locking_done;
+                } else {
+                        /* Use a copy of volname, as cli response will be
+                         * sent before the unlock, and the volname in the
+                         * dict, might be removed */
+                        volname = gf_strdup (tmp);
+                        if (!volname)
+                                goto out;
+                }
+
+                ret = glusterd_mgmt_v3_lock (volname, MY_UUID, "vol");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock for %s", volname);
+                        snprintf (err_str, err_len,
+                                  "Another transaction is in progress for %s. "
+                                  "Please try again after sometime.", volname);
+                        goto out;
+                }
+        }
+
+        locked = 1;
+        gf_log (this->name, GF_LOG_DEBUG, "Acquired lock on localhost");
+
+local_locking_done:
+
+        /* If no volname is given as a part of the command, locks will
+         * not be held, hence sending stage event. */
+        if (volname)
+                event_type = GD_OP_EVENT_START_LOCK;
+        else {
+                txn_op_info.state.state = GD_OP_STATE_LOCK_SENT;
+                event_type = GD_OP_EVENT_ALL_ACC;
+        }
+
+        /* Save opinfo for this transaction with the transaction id */
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, ctx, req);
+
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                if (ctx)
+                        dict_unref (ctx);
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, ctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to acquire cluster"
+                        " lock.");
+                goto out;
+        }
+
+out:
+        if (locked && ret) {
+                /* Based on the op-version, we release the
+                 * cluster or mgmt_v3 lock */
+                if (priv->op_version < 3)
+                        glusterd_unlock (MY_UUID);
+                else {
+                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                        ret = -1;
+                }
+        }
+
+        if (volname)
+                GF_FREE (volname);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+__glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+{
+        dict_t                         *op_ctx      = NULL;
+        int32_t                         ret         = -1;
+        gd1_mgmt_cluster_lock_req       lock_req    = {{0},};
+        glusterd_op_lock_ctx_t         *ctx         = NULL;
+        glusterd_op_t                   op          = GD_OP_EVENT_LOCK;
+        glusterd_peerinfo_t            *peerinfo    = NULL;
+        glusterd_op_info_t              txn_op_info = {{0},};
+        uuid_t                         *txn_id      = &global_txn_id;
+        xlator_t                       *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received LOCK from uuid: %s",
+                uuid_utoa (lock_req.uuid));
+
+        if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (lock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+
+        if (!ctx) {
+                //respond here
+                return -1;
+        }
+
+        uuid_copy (ctx->uuid, lock_req.uuid);
+        ctx->req = req;
+        ctx->dict = NULL;
+
+        op_ctx =  dict_new ();
+        if (!op_ctx) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set new dict");
+                goto out;
+        }
+
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, op_ctx, req);
+
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                dict_unref (txn_op_info.op_ctx);
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK, txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_LOCK");
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cluster_lock);
+}
+
+int
+glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
+                         glusterd_op_t op, uuid_t uuid,
+                         char *buf_val, size_t buf_len,
+                         gf_gld_mem_types_t mem_type,
+                         glusterd_req_ctx_t **req_ctx_out)
+{
+        int                 ret     = -1;
+        char                str[50] = {0,};
+        glusterd_req_ctx_t *req_ctx = NULL;
+        dict_t             *dict    = NULL;
+        xlator_t           *this    = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        uuid_unparse (uuid, str);
+        gf_log (this->name, GF_LOG_DEBUG, "Received op from uuid %s", str);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        req_ctx = GF_CALLOC (1, sizeof (*req_ctx), mem_type);
+        if (!req_ctx) {
+                goto out;
+        }
+
+        uuid_copy (req_ctx->uuid, uuid);
+        req_ctx->op = op;
+        ret = dict_unserialize (buf_val, buf_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        req_ctx->dict = dict;
+        req_ctx->req = rpc_req;
+        *req_ctx_out = req_ctx;
+        ret = 0;
+out:
+        if (ret) {
+                if (dict)
+                        dict_unref (dict);
+                GF_FREE (req_ctx);
+        }
+        return ret;
+}
+
+int
+__glusterd_handle_stage_op (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        glusterd_req_ctx_t              *req_ctx = NULL;
+        gd1_mgmt_stage_op_req           op_req = {{0},};
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
+        glusterd_op_info_t              txn_op_info = {{0},};
+        glusterd_op_sm_state_info_t     state;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode stage "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_req_ctx_create (req, op_req.op, op_req.uuid,
+                                       op_req.buf.buf_val, op_req.buf.buf_len,
+                                       gf_gld_mt_op_stage_ctx_t, &req_ctx);
+        if (ret)
+                goto out;
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        /* In cases where there is no volname, the receivers won't have a
+         * transaction opinfo created, as for those operations, the locking
+         * phase where the transaction opinfos are created, won't be called. */
+        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No transaction's opinfo set");
+
+                state.state = GD_OP_STATE_LOCKED;
+                glusterd_txn_opinfo_init (&txn_op_info, &state,
+                                          &op_req.op, req_ctx->dict, req);
+
+                ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set transaction's opinfo");
+                        dict_unref (req_ctx->dict);
+                        goto out;
+                }
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_OP,
+                                           txn_id, req_ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_STAGE_OP");
+
+ out:
+        free (op_req.buf.buf_val);//malloced by xdr
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+        return ret;
+}
+
+int
+glusterd_handle_stage_op (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_stage_op);
+}
+
+
+int
+__glusterd_handle_commit_op (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        glusterd_req_ctx_t              *req_ctx = NULL;
+        gd1_mgmt_commit_op_req          op_req = {{0},};
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        //the structures should always be equal
+        GF_ASSERT (sizeof (gd1_mgmt_commit_op_req) == sizeof (gd1_mgmt_stage_op_req));
+        ret = glusterd_req_ctx_create (req, op_req.op, op_req.uuid,
+                                       op_req.buf.buf_val, op_req.buf.buf_len,
+                                       gf_gld_mt_op_commit_ctx_t, &req_ctx);
+        if (ret)
+                goto out;
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_OP,
+                                           txn_id, req_ctx);
+
+out:
+        free (op_req.buf.buf_val);//malloced by xdr
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+        return ret;
+}
+
+int
+glusterd_handle_commit_op (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_commit_op);
+}
+
+int
+__glusterd_handle_cli_probe (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                  cli_req = {{0,},};
+        glusterd_peerinfo_t       *peerinfo = NULL;
+        gf_boolean_t                run_fsm = _gf_true;
+        xlator_t                      *this = NULL;
+        char                     *bind_name = NULL;
+        dict_t                        *dict = NULL;
+        char                      *hostname = NULL;
+        int                            port = 0;
+
+        GF_ASSERT (req);
+        this = THIS;
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0)  {
+                //failed to decode msg;
+                gf_log ("", GF_LOG_ERROR, "xdr decoding error");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len, &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get hostname");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "port", &port);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get port");
+                goto out;
+        }
+
+        if (glusterd_is_any_volume_in_server_quorum (this) &&
+            !does_gd_meet_server_quorum (this)) {
+                glusterd_xfer_cli_probe_resp (req, -1, GF_PROBE_QUORUM_NOT_MET,
+                                              NULL, hostname, port, dict);
+                gf_log (this->name, GF_LOG_ERROR, "Quorum does not meet, "
+                        "rejecting operation");
+                ret = 0;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO, "Received CLI probe req %s %d",
+                hostname, port);
+
+        if (dict_get_str(this->options,"transport.socket.bind-address",
+                         &bind_name) == 0) {
+                gf_log ("glusterd", GF_LOG_DEBUG,
+                        "only checking probe address vs. bind address");
+                ret = gf_is_same_address (bind_name, hostname);
+        }
+        else {
+                ret = gf_is_local_addr (hostname);
+        }
+        if (ret) {
+                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_LOCALHOST,
+                                              NULL, hostname, port, dict);
+                ret = 0;
+                goto out;
+        }
+
+        if (!(ret = glusterd_friend_find_by_hostname (hostname, &peerinfo))) {
+                if (strcmp (peerinfo->hostname, hostname) == 0) {
+
+                        gf_log ("glusterd", GF_LOG_DEBUG, "Probe host %s port "
+                                "%d already a peer", hostname, port);
+                        glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND,
+                                                      NULL, hostname, port,
+                                                      dict);
+                        goto out;
+                }
+        }
+        ret = glusterd_probe_begin (req, hostname, port, dict);
+
+        if (ret == GLUSTERD_CONNECTION_AWAITED) {
+                //fsm should be run after connection establishes
+                run_fsm = _gf_false;
+                ret = 0;
+        }
+
+out:
+        free (cli_req.dict.dict_val);
+
+        if (run_fsm) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_probe (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_cli_probe);
+}
+
+int
+__glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                  cli_req = {{0,},};
+        uuid_t                         uuid = {0};
+        int                        op_errno = 0;
+        xlator_t                      *this = NULL;
+        glusterd_conf_t               *priv = NULL;
+        dict_t                        *dict = NULL;
+        char                      *hostname = NULL;
+        int                            port = 0;
+        int                           flags = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len, &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto out;
+                }
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO, "Received CLI deprobe req");
+
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get hostname");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "port", &port);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get port");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "flags", &flags);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get flags");
+                goto out;
+        }
+
+        ret = glusterd_hostname_to_uuid (hostname, uuid);
+        if (ret) {
+                op_errno = GF_DEPROBE_NOT_FRIEND;
+                goto out;
+        }
+
+        if (!uuid_compare (uuid, MY_UUID)) {
+                op_errno = GF_DEPROBE_LOCALHOST;
+                ret = -1;
+                goto out;
+        }
+
+        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+                if (!uuid_is_null (uuid)) {
+                        /* Check if peers are connected, except peer being detached*/
+                        if (!glusterd_chk_peers_connected_befriended (uuid)) {
+                                ret = -1;
+                                op_errno = GF_DEPROBE_FRIEND_DOWN;
+                                goto out;
+                        }
+                        ret = glusterd_all_volume_cond_check (
+                                                 glusterd_friend_brick_belongs,
+                                                 -1, &uuid);
+                        if (ret) {
+                                op_errno = GF_DEPROBE_BRICK_EXIST;
+                                goto out;
+                        }
+                }
+
+                if (glusterd_is_any_volume_in_server_quorum (this) &&
+                    !does_gd_meet_server_quorum (this)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Quorum does not "
+                                "meet, rejecting operation");
+                        ret = -1;
+                        op_errno = GF_DEPROBE_QUORUM_NOT_MET;
+                        goto out;
+                }
+        }
+
+        if (!uuid_is_null (uuid)) {
+                ret = glusterd_deprobe_begin (req, hostname, port, uuid, dict);
+        } else {
+                ret = glusterd_deprobe_begin (req, hostname, port, NULL, dict);
+        }
+
+out:
+        free (cli_req.dict.dict_val);
+
+        if (ret) {
+                ret = glusterd_xfer_cli_deprobe_resp (req, ret, op_errno, NULL,
+                                                      hostname, dict);
+        }
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_cli_deprobe);
+}
+
+int
+__glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf1_cli_peer_list_req           cli_req = {0,};
+        dict_t                          *dict = NULL;
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf1_cli_peer_list_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO, "Received cli list req");
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        ret = glusterd_list_friends (req, dict, cli_req.flags);
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_list_friends);
+}
+
+int
+__glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        dict_t                          *dict = NULL;
+        int32_t                         flags = 0;
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO, "Received get vol req");
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        ret = dict_get_int32 (dict, "flags", &flags);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "failed to get flags");
+                goto out;
+        }
+
+        ret = glusterd_get_volumes (req, dict, flags);
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_get_volume);
+}
+
+int
+__glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+{
+        int                     ret     = -1;
+        dict_t                  *dict   = NULL;
+        xlator_t                *this   = NULL;
+        glusterd_conf_t         *priv   = NULL;
+        uuid_t                  uuid    = {0};
+        gf_cli_rsp              rsp     = {0,};
+        gf_cli_req              cli_req = {{0,}};
+        char                    msg_str[2048] = {0,};
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Received uuid reset req");
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
+                                  "the buffer");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        /* In the above section if dict_unserialize is successful, ret is set
+         * to zero.
+         */
+        ret = -1;
+        // Do not allow peer reset if there are any volumes in the cluster
+        if (!list_empty (&priv->volumes)) {
+                snprintf (msg_str, sizeof (msg_str), "volumes are already "
+                          "present in the cluster. Resetting uuid is not "
+                          "allowed");
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg_str);
+                goto out;
+        }
+
+        // Do not allow peer reset if trusted storage pool is already formed
+        if (!list_empty (&priv->peers)) {
+                snprintf (msg_str, sizeof (msg_str),"trusted storage pool "
+                          "has been already formed. Please detach this peer "
+                          "from the pool and reset its uuid.");
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg_str);
+                goto out;
+        }
+
+        uuid_copy (uuid, priv->uuid);
+        ret = glusterd_uuid_generate_save ();
+
+        if (!uuid_compare (uuid, MY_UUID)) {
+                snprintf (msg_str, sizeof (msg_str), "old uuid and the new uuid"
+                          " are same. Try gluster peer reset again");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg_str);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        if (ret) {
+                rsp.op_ret = -1;
+                if (msg_str[0] == '\0')
+                        snprintf (msg_str, sizeof (msg_str), "Operation "
+                                  "failed");
+                rsp.op_errstr = msg_str;
+                ret = 0;
+        } else {
+                rsp.op_errstr = "";
+        }
+
+        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
+                         (xdrproc_t)xdr_gf_cli_rsp, dict);
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_uuid_reset);
+}
+
+int
+__glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+{
+        int                     ret         = -1;
+        dict_t                  *dict       = NULL;
+        dict_t                  *rsp_dict   = NULL;
+        xlator_t                *this       = NULL;
+        glusterd_conf_t         *priv       = NULL;
+        gf_cli_rsp              rsp         = {0,};
+        gf_cli_req              cli_req     = {{0,}};
+        char                    msg_str[2048] = {0,};
+        char                    uuid_str[64] = {0,};
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Received uuid get req");
+
+        if (cli_req.dict.dict_len) {
+                dict  = dict_new ();
+                if (!dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
+                                  "the buffer");
+                        goto out;
+
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+
+                }
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+        ret = dict_set_str (rsp_dict, "uuid", uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set uuid in "
+                        "dictionary.");
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize "
+                        "dictionary.");
+                goto out;
+        }
+        ret = 0;
+out:
+        if (ret) {
+                rsp.op_ret = -1;
+                if (msg_str[0] == '\0')
+                        snprintf (msg_str, sizeof (msg_str), "Operation "
+                                  "failed");
+                rsp.op_errstr = msg_str;
+
+        } else {
+                rsp.op_errstr = "";
+
+        }
+
+        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
+                         (xdrproc_t)xdr_gf_cli_rsp, dict);
+
+        return 0;
+}
+int
+glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_uuid_get);
+}
+
+int
+__glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+{
+        int                     ret = -1;
+        dict_t                  *dict = NULL;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        int                     count = 0;
+        char                    key[1024] = {0,};
+        gf_cli_rsp              rsp = {0,};
+
+        GF_ASSERT (req);
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d", count);
+                ret = dict_set_str (dict, key, volinfo->volname);
+                if (ret)
+                        goto out;
+                count++;
+        }
+
+        ret = dict_set_int32 (dict, "count", count);
+        if (ret)
+                goto out;
+
+        ret = dict_allocate_and_serialize (dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret)
+                goto out;
+
+        ret = 0;
+
+out:
+        rsp.op_ret = ret;
+        if (ret)
+                rsp.op_errstr = "Error listing volumes";
+        else
+                rsp.op_errstr = "";
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gf_cli_rsp);
+        ret = 0;
+
+        if (dict)
+                dict_unref (dict);
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_list_volume);
+}
+
+int32_t
+glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                   char *err_str, size_t err_len)
+{
+        int             ret = -1;
+
+        ret = glusterd_op_txn_begin (req, op, ctx, err_str, err_len);
+
+        return ret;
+}
+
+int
+__glusterd_handle_reset_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        dict_t                          *dict = NULL;
+        glusterd_op_t                   cli_op = GD_OP_RESET_VOLUME;
+        char                            *volname = NULL;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode request "
+                          "received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                    "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Received volume reset request for "
+                "volume %s", volname);
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_RESET_VOLUME, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_reset_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_reset_volume);
+}
+
+int
+__glusterd_handle_set_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        dict_t                          *dict = NULL;
+        glusterd_op_t                   cli_op = GD_OP_SET_VOLUME;
+        char                            *key = NULL;
+        char                            *value = NULL;
+        char                            *volname = NULL;
+        char                            *op_errstr = NULL;
+        gf_boolean_t                    help = _gf_false;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode "
+                          "request received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name while handling volume set command");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        if (strcmp (volname, "help") == 0 ||
+            strcmp (volname, "help-xml") == 0) {
+                ret = glusterd_volset_help (dict, &op_errstr);
+                help = _gf_true;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "key1", &key);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to get key while"
+                          " handling volume set for %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "value1", &value);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to get value while"
+                          " handling volume set for %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Received volume set request for "
+                "volume %s", volname);
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_SET_VOLUME, dict);
+
+out:
+        if (help)
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req, dict,
+                                                     (op_errstr)? op_errstr:"");
+        else if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        if (op_errstr)
+                GF_FREE (op_errstr);
+
+        return ret;
+}
+
+int
+glusterd_handle_set_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_set_volume);
+}
+
+int
+__glusterd_handle_sync_volume (rpcsvc_request_t *req)
+{
+        int32_t                          ret     = -1;
+        gf_cli_req                       cli_req = {{0,}};
+        dict_t                           *dict = NULL;
+        gf_cli_rsp                       cli_rsp = {0.};
+        char                             msg[2048] = {0,};
+        char                             *volname = NULL;
+        gf1_cli_sync_volume              flags = 0;
+        char                             *hostname = NULL;
+        xlator_t                         *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to get hostname");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                ret = dict_get_int32 (dict, "flags", (int32_t*)&flags);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), "Failed to get volume name"
+                                  " or flags");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                        goto out;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Received volume sync req "
+                "for volume %s", (flags & GF_CLI_SYNC_ALL) ? "all" : volname);
+
+        if (gf_is_local_addr (hostname)) {
+                ret = -1;
+                snprintf (msg, sizeof (msg), "sync from localhost"
+                          " not allowed");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_SYNC_VOLUME, dict);
+
+out:
+        if (ret) {
+                cli_rsp.op_ret = -1;
+                cli_rsp.op_errstr = msg;
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
+                glusterd_to_cli (req, &cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
+
+                ret = 0; //sent error to cli, prevent second reply
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_sync_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_sync_volume);
+}
+
+int
+glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
+                            char *op_errstr, dict_t *dict)
+{
+
+        int                             ret = -1;
+        gf1_cli_fsm_log_rsp             rsp = {0};
+
+        GF_ASSERT (req);
+        GF_ASSERT (op_errstr);
+
+        rsp.op_ret = op_ret;
+        rsp.op_errstr = op_errstr;
+        if (rsp.op_ret == 0)
+                ret = dict_allocate_and_serialize (dict, &rsp.fsm_log.fsm_log_val,
+                                                &rsp.fsm_log.fsm_log_len);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
+        GF_FREE (rsp.fsm_log.fsm_log_val);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Responded, ret: %d", ret);
+
+        return 0;
+}
+
+int
+__glusterd_handle_fsm_log (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf1_cli_fsm_log_req             cli_req = {0,};
+        dict_t                          *dict = NULL;
+        glusterd_sm_tr_log_t            *log = NULL;
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *conf = NULL;
+        char                            msg[2048] = {0};
+        glusterd_peerinfo_t             *peerinfo = NULL;
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf1_cli_fsm_log_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                snprintf (msg, sizeof (msg), "Garbage request");
+                goto out;
+        }
+
+        if (strcmp ("", cli_req.name) == 0) {
+                this = THIS;
+                conf = this->private;
+                log = &conf->op_sm_log;
+        } else {
+                ret = glusterd_friend_find_by_hostname (cli_req.name,
+                                                        &peerinfo);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), "%s is not a peer",
+                                  cli_req.name);
+                        goto out;
+                }
+                log = &peerinfo->sm_log;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_sm_tr_log_add_to_dict (dict, log);
+out:
+        (void)glusterd_fsm_log_send_resp (req, ret, msg, dict);
+        free (cli_req.name);//malloced by xdr
+        if (dict)
+                dict_unref (dict);
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return 0;//send 0 to avoid double reply
+}
+
+int
+glusterd_handle_fsm_log (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_fsm_log);
+}
+
+int
+glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status)
+{
+
+        gd1_mgmt_cluster_lock_rsp       rsp = {{0},};
+        int                             ret = -1;
+
+        GF_ASSERT (req);
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op_ret = status;
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to lock, ret: %d", ret);
+
+        return 0;
+}
+
+int
+glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
+{
+
+        gd1_mgmt_cluster_unlock_rsp     rsp = {{0},};
+        int                             ret = -1;
+
+        GF_ASSERT (req);
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to unlock, ret: %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_op_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+                                    int32_t status)
+{
+
+        gd1_mgmt_v3_lock_rsp           rsp = {{0},};
+        int                            ret = -1;
+
+        GF_ASSERT (req);
+        GF_ASSERT (txn_id);
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+        uuid_copy (rsp.txn_id, *txn_id);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to mgmt_v3 lock, ret: %d",
+                ret);
+
+        return ret;
+}
+
+int
+glusterd_op_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+                                      int32_t status)
+{
+
+        gd1_mgmt_v3_unlock_rsp          rsp = {{0},};
+        int                             ret = -1;
+
+        GF_ASSERT (req);
+        GF_ASSERT (txn_id);
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+        glusterd_get_uuid (&rsp.uuid);
+        uuid_copy (rsp.txn_id, *txn_id);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG,
+                "Responded to mgmt_v3 unlock, ret: %d",
+                ret);
+
+        return ret;
+}
+
+int
+__glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+{
+        gd1_mgmt_cluster_unlock_req     unlock_req = {{0}, };
+        int32_t                         ret = -1;
+        glusterd_op_lock_ctx_t          *ctx = NULL;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        xlator_t                        *this = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &unlock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Received UNLOCK from uuid: %s", uuid_utoa (unlock_req.uuid));
+
+        if (glusterd_friend_find_by_uuid (unlock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (unlock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+
+        if (!ctx) {
+                //respond here
+                return -1;
+        }
+        uuid_copy (ctx->uuid, unlock_req.uuid);
+        ctx->req = req;
+        ctx->dict = NULL;
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK, txn_id, ctx);
+
+out:
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cluster_unlock);
+}
+
+int
+glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
+                             int32_t op, int32_t status,
+                             char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_stage_op_rsp           rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                return ret;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to stage, ret: %d", ret);
+        GF_FREE (rsp.dict.dict_val);
+
+        return ret;
+}
+
+int
+glusterd_op_commit_send_resp (rpcsvc_request_t *req,
+                               int32_t op, int32_t status, char *op_errstr,
+                               dict_t *rsp_dict)
+{
+        gd1_mgmt_commit_op_rsp          rsp      = {{0}, };
+        int                             ret      = -1;
+        xlator_t                        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        if (rsp_dict) {
+                ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                                   &rsp.dict.dict_len);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to get serialized length of dict");
+                        goto out;
+                }
+        }
+
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to commit, ret: %d", ret);
+
+out:
+        GF_FREE (rsp.dict.dict_val);
+        return ret;
+}
+
+int
+__glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+{
+        int32_t                 ret = -1;
+        gd1_mgmt_friend_req     friend_req = {{0},};
+        gf_boolean_t            run_fsm = _gf_true;
+
+        GF_ASSERT (req);
+        ret = xdr_to_generic (req->msg[0], &friend_req,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO,
+                "Received probe from uuid: %s", uuid_utoa (friend_req.uuid));
+        ret = glusterd_handle_friend_req (req, friend_req.uuid,
+                                          friend_req.hostname, friend_req.port,
+                                          &friend_req);
+
+        if (ret == GLUSTERD_CONNECTION_AWAITED) {
+                //fsm should be run after connection establishes
+                run_fsm = _gf_false;
+                ret = 0;
+        }
+
+out:
+        free (friend_req.hostname);//malloced by xdr
+
+        if (run_fsm) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_incoming_friend_req);
+}
+
+int
+__glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+{
+        int32_t                 ret = -1;
+        gd1_mgmt_friend_req     friend_req = {{0},};
+        char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
+
+        GF_ASSERT (req);
+        ret = xdr_to_generic (req->msg[0], &friend_req,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO,
+                "Received unfriend from uuid: %s", uuid_utoa (friend_req.uuid));
+
+        ret = glusterd_remote_hostname_get (req, remote_hostname,
+                                            sizeof (remote_hostname));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get the remote hostname");
+                goto out;
+        }
+        ret = glusterd_handle_unfriend_req (req, friend_req.uuid,
+                                            remote_hostname, friend_req.port);
+
+out:
+        free (friend_req.hostname);//malloced by xdr
+        free (friend_req.vols.vols_val);//malloced by xdr
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_incoming_unfriend_req);
+
+}
+
+int
+glusterd_handle_friend_update_delete (dict_t *dict)
+{
+        char                    *hostname = NULL;
+        int32_t                 ret = -1;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret)
+                goto out;
+
+        ret = glusterd_friend_remove (NULL, hostname);
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_friend_hostname_update (glusterd_peerinfo_t *peerinfo,
+                                char *hostname,
+                                gf_boolean_t store_update)
+{
+        char                    *new_hostname = NULL;
+        int                     ret = 0;
+
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (hostname);
+
+        new_hostname = gf_strdup (hostname);
+        if (!new_hostname) {
+                ret = -1;
+                goto out;
+        }
+
+        GF_FREE (peerinfo->hostname);
+        peerinfo->hostname = new_hostname;
+        if (store_update)
+                ret = glusterd_store_peerinfo (peerinfo);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+__glusterd_handle_friend_update (rpcsvc_request_t *req)
+{
+        int32_t                 ret = -1;
+        gd1_mgmt_friend_update     friend_req = {{0},};
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        glusterd_conf_t         *priv = NULL;
+        xlator_t                *this = NULL;
+        glusterd_peerinfo_t     *tmp = NULL;
+        gd1_mgmt_friend_update_rsp rsp = {{0},};
+        dict_t                  *dict = NULL;
+        char                    key[100] = {0,};
+        char                    *uuid_buf = NULL;
+        char                    *hostname = NULL;
+        int                     i = 1;
+        int                     count = 0;
+        uuid_t                  uuid = {0,};
+        glusterd_peerctx_args_t args = {0};
+        int32_t                 op = 0;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &friend_req,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_update);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        ret = glusterd_friend_find (friend_req.uuid, NULL, &tmp);
+        if (ret) {
+                gf_log ("", GF_LOG_CRITICAL, "Received friend update request "
+                        "from unknown peer %s", uuid_utoa (friend_req.uuid));
+                goto out;
+        }
+        gf_log ("glusterd", GF_LOG_INFO,
+                "Received friend update from uuid: %s", uuid_utoa (friend_req.uuid));
+
+        if (friend_req.friends.friends_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (friend_req.friends.friends_val,
+                                        friend_req.friends.friends_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = friend_req.friends.friends_val;
+                }
+        }
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, "op", &op);
+        if (ret)
+                goto out;
+
+        if (GD_FRIEND_UPDATE_DEL == op) {
+                ret = glusterd_handle_friend_update_delete (dict);
+                goto out;
+        }
+
+        args.mode = GD_MODE_ON;
+        while ( i <= count) {
+                snprintf (key, sizeof (key), "friend%d.uuid", i);
+                ret = dict_get_str (dict, key, &uuid_buf);
+                if (ret)
+                        goto out;
+                uuid_parse (uuid_buf, uuid);
+                snprintf (key, sizeof (key), "friend%d.hostname", i);
+                ret = dict_get_str (dict, key, &hostname);
+                if (ret)
+                        goto out;
+
+                gf_log ("", GF_LOG_INFO, "Received uuid: %s, hostname:%s",
+                                uuid_buf, hostname);
+
+                if (uuid_is_null (uuid)) {
+                        gf_log (this->name, GF_LOG_WARNING, "Updates mustn't "
+                                "contain peer with 'null' uuid");
+                        continue;
+                }
+
+                if (!uuid_compare (uuid, MY_UUID)) {
+                        gf_log ("", GF_LOG_INFO, "Received my uuid as Friend");
+                        i++;
+                        continue;
+                }
+
+                ret = glusterd_friend_find (uuid, hostname, &tmp);
+
+                if (!ret) {
+                        if (strcmp (hostname, tmp->hostname) != 0) {
+                                glusterd_friend_hostname_update (tmp, hostname,
+                                                                 _gf_true);
+                        }
+                        i++;
+                        continue;
+                }
+
+                ret = glusterd_friend_add (hostname, friend_req.port,
+                                           GD_FRIEND_STATE_BEFRIENDED,
+                                           &uuid, &peerinfo, 0, &args);
+
+                i++;
+        }
+
+out:
+        uuid_copy (rsp.uuid, MY_UUID);
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
+        if (dict) {
+                if (!dict->extra_stdfree && friend_req.friends.friends_val)
+                        free (friend_req.friends.friends_val);//malloced by xdr
+                dict_unref (dict);
+        } else {
+                free (friend_req.friends.friends_val);//malloced by xdr
+        }
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_friend_update (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_friend_update);
+}
+
+int
+__glusterd_handle_probe_query (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *conf = NULL;
+        gd1_mgmt_probe_req              probe_req = {{0},};
+        gd1_mgmt_probe_rsp              rsp = {{0},};
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_peerctx_args_t         args = {0};
+        int                             port = 0;
+        char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &probe_req,
+                              (xdrproc_t)xdr_gd1_mgmt_probe_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        this = THIS;
+
+        conf = this->private;
+        if (probe_req.port)
+                port = probe_req.port;
+        else
+                port = GF_DEFAULT_BASE_PORT;
+
+        gf_log ("glusterd", GF_LOG_INFO,
+                "Received probe from uuid: %s", uuid_utoa (probe_req.uuid));
+
+        /* Check for uuid collision and handle it in a user friendly way by
+         * sending the error.
+         */
+        if (!uuid_compare (probe_req.uuid, MY_UUID)) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Peer uuid %s is same as "
+                        "local uuid. Please check the uuid of both the peers "
+                        "from %s/%s", uuid_utoa (probe_req.uuid),
+                        GLUSTERD_DEFAULT_WORKDIR, GLUSTERD_INFO_FILE);
+                rsp.op_ret = -1;
+                rsp.op_errno = GF_PROBE_SAME_UUID;
+                rsp.port = port;
+                goto respond;
+        }
+
+        ret = glusterd_remote_hostname_get (req, remote_hostname,
+                                            sizeof (remote_hostname));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get the remote hostname");
+                goto out;
+        }
+        ret = glusterd_friend_find (probe_req.uuid, remote_hostname, &peerinfo);
+        if ((ret != 0 ) && (!list_empty (&conf->peers))) {
+                rsp.op_ret = -1;
+                rsp.op_errno = GF_PROBE_ANOTHER_CLUSTER;
+        } else if (ret) {
+                gf_log ("glusterd", GF_LOG_INFO, "Unable to find peerinfo"
+                        " for host: %s (%d)", remote_hostname, port);
+                args.mode = GD_MODE_ON;
+                ret = glusterd_friend_add (remote_hostname, port,
+                                           GD_FRIEND_STATE_PROBE_RCVD,
+                                           NULL, &peerinfo, 0, &args);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to add peer %s",
+                                remote_hostname);
+                        rsp.op_errno = GF_PROBE_ADD_FAILED;
+                }
+        }
+
+respond:
+        uuid_copy (rsp.uuid, MY_UUID);
+
+        rsp.hostname = probe_req.hostname;
+        rsp.op_errstr = "";
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+        ret = 0;
+
+        gf_log ("glusterd", GF_LOG_INFO, "Responded to %s, op_ret: %d, "
+                "op_errno: %d, ret: %d", remote_hostname,
+                rsp.op_ret, rsp.op_errno, ret);
+
+out:
+        free (probe_req.hostname);//malloced by xdr
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int glusterd_handle_probe_query (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_probe_query);
+}
+
+int
+__glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret     = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        dict_t                          *dict = NULL;
+        glusterd_op_t                   cli_op = GD_OP_PROFILE_VOLUME;
+        char                            *volname = NULL;
+        int32_t                         op = 0;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len > 0) {
+                dict = dict_new();
+                if (!dict)
+                        goto out;
+                dict_unserialize (cli_req.dict.dict_val,
+                                  cli_req.dict.dict_len, &dict);
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Received volume profile req "
+                "for volume %s", volname);
+        ret = dict_get_int32 (dict, "op", &op);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get operation");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = glusterd_op_begin (req, cli_op, dict, err_str, sizeof (err_str));
+
+out:
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        free (cli_req.dict.dict_val);
+
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_profile_volume);
+}
+
+int
+__glusterd_handle_getwd (rpcsvc_request_t *req)
+{
+        int32_t                 ret = -1;
+        gf1_cli_getwd_rsp     rsp = {0,};
+        glusterd_conf_t         *priv = NULL;
+
+        GF_ASSERT (req);
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        gf_log ("glusterd", GF_LOG_INFO, "Received getwd req");
+
+        rsp.wd = priv->workdir;
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+        ret = 0;
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_getwd (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_getwd);
+}
+
+int
+__glusterd_handle_mount (rpcsvc_request_t *req)
+{
+        gf1_cli_mount_req mnt_req = {0,};
+        gf1_cli_mount_rsp rsp     = {0,};
+        dict_t *dict              = NULL;
+        int ret                   = 0;
+        glusterd_conf_t     *priv   = NULL;
+
+        GF_ASSERT (req);
+	priv = THIS->private;
+
+        ret = xdr_to_generic (req->msg[0], &mnt_req,
+                              (xdrproc_t)xdr_gf1_cli_mount_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                rsp.op_ret = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO, "Received mount req");
+
+        if (mnt_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (mnt_req.dict.dict_val,
+                                        mnt_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        rsp.op_ret = -1;
+                        rsp.op_errno = -EINVAL;
+                        goto out;
+                } else {
+                        dict->extra_stdfree = mnt_req.dict.dict_val;
+                }
+        }
+
+	synclock_unlock (&priv->big_lock);
+        rsp.op_ret = glusterd_do_mount (mnt_req.label, dict,
+                                        &rsp.path, &rsp.op_errno);
+	synclock_lock (&priv->big_lock);
+
+ out:
+        if (!rsp.path)
+                rsp.path = "";
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_mount_rsp);
+        ret = 0;
+
+        if (dict)
+                dict_unref (dict);
+        if (*rsp.path)
+                GF_FREE (rsp.path);
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_mount (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_mount);
+}
+
+int
+__glusterd_handle_umount (rpcsvc_request_t *req)
+{
+        gf1_cli_umount_req umnt_req = {0,};
+        gf1_cli_umount_rsp rsp      = {0,};
+        char *mountbroker_root      = NULL;
+        char mntp[PATH_MAX]         = {0,};
+        char *path                  = NULL;
+        runner_t runner             = {0,};
+        int ret                     = 0;
+        xlator_t *this              = THIS;
+        gf_boolean_t dir_ok         = _gf_false;
+        char *pdir                  = NULL;
+        char *t                     = NULL;
+        glusterd_conf_t     *priv   = NULL;
+
+        GF_ASSERT (req);
+        GF_ASSERT (this);
+        priv = this->private;
+
+        ret = xdr_to_generic (req->msg[0], &umnt_req,
+                              (xdrproc_t)xdr_gf1_cli_umount_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                rsp.op_ret = -1;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO, "Received umount req");
+
+        if (dict_get_str (this->options, "mountbroker-root",
+                          &mountbroker_root) != 0) {
+                rsp.op_errno = ENOENT;
+                goto out;
+        }
+
+        /* check if it is allowed to umount path */
+        path = gf_strdup (umnt_req.path);
+        if (!path) {
+                rsp.op_errno = ENOMEM;
+                goto out;
+        }
+        dir_ok = _gf_false;
+        pdir = dirname (path);
+        t = strtail (pdir, mountbroker_root);
+        if (t && *t == '/') {
+                t = strtail(++t, MB_HIVE);
+                if (t && !*t)
+                        dir_ok = _gf_true;
+        }
+        GF_FREE (path);
+        if (!dir_ok) {
+                rsp.op_errno = EACCES;
+                goto out;
+        }
+
+        runinit (&runner);
+        runner_add_args (&runner, "umount", umnt_req.path, NULL);
+        if (umnt_req.lazy)
+                runner_add_arg (&runner, "-l");
+        synclock_unlock (&priv->big_lock);
+        rsp.op_ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (rsp.op_ret == 0) {
+                if (realpath (umnt_req.path, mntp))
+                        rmdir (mntp);
+                else {
+                        rsp.op_ret = -1;
+                        rsp.op_errno = errno;
+                }
+                if (unlink (umnt_req.path) != 0) {
+                        rsp.op_ret = -1;
+                        rsp.op_errno = errno;
+                }
+        }
+
+ out:
+        if (rsp.op_errno)
+                rsp.op_ret = -1;
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_umount_rsp);
+        ret = 0;
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        return ret;
+}
+
+int
+glusterd_handle_umount (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_umount);
+}
+
+int
+glusterd_friend_remove (uuid_t uuid, char *hostname)
+{
+        int                           ret = 0;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+
+        ret = glusterd_friend_find (uuid, hostname, &peerinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_friend_remove_cleanup_vols (peerinfo->uuid);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING, "Volumes cleanup failed");
+        ret = glusterd_friend_cleanup (peerinfo);
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_rpc_create (struct rpc_clnt **rpc,
+                     dict_t *options,
+                     rpc_clnt_notify_t notify_fn,
+                     void *notify_data)
+{
+        struct rpc_clnt         *new_rpc = NULL;
+        int                     ret = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (options);
+
+        /* TODO: is 32 enough? or more ? */
+        new_rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
+        if (!new_rpc)
+                goto out;
+
+        ret = rpc_clnt_register_notify (new_rpc, notify_fn, notify_data);
+        *rpc = new_rpc;
+        if (ret)
+                goto out;
+        ret = rpc_clnt_start (new_rpc);
+out:
+        if (ret) {
+                if (new_rpc) {
+                        (void) rpc_clnt_unref (new_rpc);
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_transport_keepalive_options_get (int *interval, int *time)
+{
+        int     ret = 0;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_int32 (this->options,
+                              "transport.socket.keepalive-interval",
+                              interval);
+        ret = dict_get_int32 (this->options,
+                              "transport.socket.keepalive-time",
+                              time);
+        return 0;
+}
+
+int
+glusterd_transport_inet_options_build (dict_t **options, const char *hostname,
+                                       int port)
+{
+        dict_t  *dict = NULL;
+        int32_t interval = -1;
+        int32_t time     = -1;
+        int     ret = 0;
+
+        GF_ASSERT (options);
+        GF_ASSERT (hostname);
+
+        if (!port)
+                port = GLUSTERD_DEFAULT_PORT;
+
+        /* Build default transport options */
+        ret = rpc_transport_inet_options_build (&dict, hostname, port);
+        if (ret)
+                goto out;
+
+        /* Set frame-timeout to 10mins. Default timeout of 30 mins is too long
+         * when compared to 2 mins for cli timeout. This ensures users don't
+         * wait too long after cli timesout before being able to resume normal
+         * operations
+         */
+        ret = dict_set_int32 (dict, "frame-timeout", 600);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "Failed to set frame-timeout");
+                goto out;
+        }
+
+        /* Set keepalive options */
+        glusterd_transport_keepalive_options_get (&interval, &time);
+
+        if ((interval > 0) || (time > 0))
+                ret = rpc_transport_keepalive_options_set (dict, interval, time);
+        *options = dict;
+out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                            glusterd_peerctx_args_t *args)
+{
+        dict_t                 *options = NULL;
+        int                    ret = -1;
+        glusterd_peerctx_t     *peerctx = NULL;
+        data_t                 *data = NULL;
+
+        peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t);
+        if (!peerctx)
+                goto out;
+
+        if (args)
+                peerctx->args = *args;
+
+        peerctx->peerinfo = peerinfo;
+
+        ret = glusterd_transport_inet_options_build (&options,
+                                                     peerinfo->hostname,
+                                                     peerinfo->port);
+        if (ret)
+                goto out;
+
+        /*
+         * For simulated multi-node testing, we need to make sure that we
+         * create our RPC endpoint with the same address that the peer would
+         * use to reach us.
+         */
+        if (this->options) {
+                data = dict_get(this->options,"transport.socket.bind-address");
+                if (data) {
+                        ret = dict_set(options,
+                                       "transport.socket.source-addr",data);
+                }
+        }
+
+        ret = glusterd_rpc_create (&peerinfo->rpc, options,
+                                   glusterd_peer_rpc_notify, peerctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to create rpc for"
+                        " peer %s", peerinfo->hostname);
+                goto out;
+        }
+        peerctx = NULL;
+        ret = 0;
+out:
+        GF_FREE (peerctx);
+        return ret;
+}
+
+int
+glusterd_friend_add (const char *hoststr, int port,
+                     glusterd_friend_sm_state_t state,
+                     uuid_t *uuid,
+                     glusterd_peerinfo_t **friend,
+                     gf_boolean_t restore,
+                     glusterd_peerctx_args_t *args)
+{
+        int                     ret = 0;
+        xlator_t               *this = NULL;
+        glusterd_conf_t        *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+        GF_ASSERT (hoststr);
+
+        ret = glusterd_peerinfo_new (friend, state, uuid, hoststr, port);
+        if (ret) {
+                goto out;
+        }
+
+        /*
+         * We can't add to the list after calling glusterd_friend_rpc_create,
+         * even if it succeeds, because by then the callback to take it back
+         * off and free might have happened already (notably in the case of an
+         * invalid peer name).  That would mean we're adding something that had
+         * just been free, and we're likely to crash later.
+         */
+        list_add_tail (&(*friend)->uuid_list, &conf->peers);
+
+        //restore needs to first create the list of peers, then create rpcs
+        //to keep track of quorum in race-free manner. In restore for each peer
+        //rpc-create calls rpc_notify when the friend-list is partially
+        //constructed, leading to wrong quorum calculations.
+        if (!restore) {
+                ret = glusterd_store_peerinfo (*friend);
+                if (ret == 0) {
+                        synclock_unlock (&conf->big_lock);
+                        ret = glusterd_friend_rpc_create (this, *friend, args);
+                        synclock_lock (&conf->big_lock);
+                }
+                else {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store peerinfo");
+                }
+        }
+
+        if (ret) {
+                (void) glusterd_friend_cleanup (*friend);
+                *friend = NULL;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_INFO, "connect returned %d", ret);
+        return ret;
+}
+
+int
+glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
+                      dict_t *dict)
+{
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_peerctx_args_t         args = {0};
+        glusterd_friend_sm_event_t      *event = NULL;
+
+        GF_ASSERT (hoststr);
+
+        ret = glusterd_friend_find (NULL, (char *)hoststr, &peerinfo);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_INFO, "Unable to find peerinfo"
+                        " for host: %s (%d)", hoststr, port);
+                args.mode = GD_MODE_ON;
+                args.req  = req;
+                args.dict = dict;
+                ret = glusterd_friend_add ((char *)hoststr, port,
+                                           GD_FRIEND_STATE_DEFAULT,
+                                           NULL, &peerinfo, 0, &args);
+                if ((!ret) && (!peerinfo->connected)) {
+                        ret = GLUSTERD_CONNECTION_AWAITED;
+                }
+
+        } else if (peerinfo->connected &&
+                   (GD_FRIEND_STATE_BEFRIENDED == peerinfo->state.state)) {
+                ret = glusterd_friend_hostname_update (peerinfo, (char*)hoststr,
+                                                       _gf_false);
+                if (ret)
+                        goto out;
+                //this is just to rename so inject local acc for cluster update
+                ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_LOCAL_ACC,
+                                                    &event);
+                if (!ret) {
+                        event->peerinfo = peerinfo;
+                        ret = glusterd_friend_sm_inject_event (event);
+                        glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_SUCCESS,
+                                                      NULL, (char*)hoststr,
+                                                      port, dict);
+                }
+        } else {
+                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND, NULL,
+                                              (char*)hoststr, port, dict);
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
+                        uuid_t uuid, dict_t *dict)
+{
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_friend_sm_event_t      *event = NULL;
+        glusterd_probe_ctx_t            *ctx = NULL;
+
+        GF_ASSERT (hoststr);
+        GF_ASSERT (req);
+
+        ret = glusterd_friend_find (uuid, (char *)hoststr, &peerinfo);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_INFO, "Unable to find peerinfo"
+                        " for host: %s %d", hoststr, port);
+                goto out;
+        }
+
+        if (!peerinfo->rpc) {
+                //handle this case
+                goto out;
+        }
+
+        ret = glusterd_friend_sm_new_event
+                (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND, &event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                                "Unable to get new event");
+                return ret;
+        }
+
+        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
+
+        if (!ctx) {
+                goto out;
+        }
+
+        ctx->hostname = gf_strdup (hoststr);
+        ctx->port = port;
+        ctx->req = req;
+        ctx->dict = dict;
+
+        event->ctx = ctx;
+
+        event->peerinfo = peerinfo;
+
+        ret = glusterd_friend_sm_inject_event (event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject event %d, "
+                        "ret = %d", event->event, ret);
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+
+int
+glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int port)
+{
+        gd1_mgmt_friend_rsp  rsp = {{0}, };
+        int32_t              ret = -1;
+        xlator_t             *this = NULL;
+        glusterd_conf_t      *conf = NULL;
+
+        GF_ASSERT (hostname);
+
+        rsp.op_ret = 0;
+        this = THIS;
+        GF_ASSERT (this);
+
+        conf = this->private;
+
+        uuid_copy (rsp.uuid, MY_UUID);
+        rsp.hostname = hostname;
+        rsp.port = port;
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+
+        gf_log ("glusterd", GF_LOG_INFO,
+                "Responded to %s (%d), ret: %d", hostname, port, ret);
+        return ret;
+}
+
+
+int
+glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
+                               char *remote_hostname, int port, int32_t op_ret,
+                               int32_t op_errno)
+{
+        gd1_mgmt_friend_rsp  rsp = {{0}, };
+        int32_t              ret = -1;
+        xlator_t             *this = NULL;
+        glusterd_conf_t      *conf = NULL;
+
+        GF_ASSERT (myhostname);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        conf = this->private;
+
+        uuid_copy (rsp.uuid, MY_UUID);
+        rsp.op_ret = op_ret;
+        rsp.op_errno = op_errno;
+        rsp.hostname = gf_strdup (myhostname);
+        rsp.port = port;
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+
+        gf_log ("glusterd", GF_LOG_INFO,
+                "Responded to %s (%d), ret: %d", remote_hostname, port, ret);
+        GF_FREE (rsp.hostname);
+        return ret;
+}
+
+static void
+set_probe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
+                     size_t len, char *hostname, int port)
+{
+        if ((op_errstr) && (strcmp (op_errstr, ""))) {
+                snprintf (errstr, len, "%s", op_errstr);
+                return;
+        }
+
+        if (!op_ret) {
+                switch (op_errno) {
+                        case GF_PROBE_LOCALHOST:
+                                snprintf (errstr, len, "Probe on localhost not "
+                                          "needed");
+                                break;
+
+                        case GF_PROBE_FRIEND:
+                                snprintf (errstr, len, "Host %s port %d already"
+                                          " in peer list", hostname, port);
+                                break;
+
+                        default:
+                                if (op_errno != 0)
+                                        snprintf (errstr, len, "Probe returned "
+                                                  "with unknown errno %d",
+                                                  op_errno);
+                                break;
+                }
+        } else {
+                switch (op_errno) {
+                        case GF_PROBE_ANOTHER_CLUSTER:
+                                snprintf (errstr, len, "%s is already part of "
+                                          "another cluster", hostname);
+                                break;
+
+                        case GF_PROBE_VOLUME_CONFLICT:
+                                snprintf (errstr, len, "Atleast one volume on "
+                                          "%s conflicts with existing volumes "
+                                          "in the cluster", hostname);
+                                break;
+
+                        case GF_PROBE_UNKNOWN_PEER:
+                                snprintf (errstr, len, "%s responded with "
+                                          "'unknown peer' error, this could "
+                                          "happen if %s doesn't have localhost "
+                                          "in its peer database", hostname,
+                                          hostname);
+                                break;
+
+                        case GF_PROBE_ADD_FAILED:
+                                snprintf (errstr, len, "Failed to add peer "
+                                          "information on %s", hostname);
+                                break;
+
+                        case GF_PROBE_SAME_UUID:
+                                snprintf (errstr, len, "Peer uuid (host %s) is "
+                                          "same as local uuid", hostname);
+                                break;
+
+                        case GF_PROBE_QUORUM_NOT_MET:
+                                snprintf (errstr, len, "Cluster quorum is not "
+                                          "met. Changing peers is not allowed "
+                                          "in this state");
+                                break;
+
+                        default:
+                                snprintf (errstr, len, "Probe returned with "
+                                          "unknown errno %d", op_errno);
+                                break;
+                }
+        }
+}
+
+int
+glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
+                              int32_t op_errno, char *op_errstr, char *hostname,
+                              int port, dict_t *dict)
+{
+        gf_cli_rsp           rsp = {0,};
+        int32_t              ret = -1;
+        char        errstr[2048] = {0,};
+        char            *cmd_str = NULL;
+        xlator_t           *this = THIS;
+
+        GF_ASSERT (req);
+        GF_ASSERT (this);
+
+        (void) set_probe_error_str (op_ret, op_errno, op_errstr, errstr,
+                                    sizeof (errstr), hostname, port);
+
+        if (dict) {
+                ret = dict_get_str (dict, "cmd-str", &cmd_str);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "command string");
+        }
+
+        rsp.op_ret = op_ret;
+        rsp.op_errno = op_errno;
+        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+
+        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
+                    (op_ret) ? "FAILED" : "SUCCESS",
+                    (errstr[0] != '\0') ? ":" : " ",
+                    (errstr[0] != '\0') ? errstr : " ");
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gf_cli_rsp);
+
+        if (dict)
+                dict_unref (dict);
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to CLI, ret: %d",ret);
+
+        return ret;
+}
+
+static void
+set_deprobe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
+                       size_t len, char *hostname)
+{
+        if ((op_errstr) && (strcmp (op_errstr, ""))) {
+                snprintf (errstr, len, "%s", op_errstr);
+                return;
+        }
+
+        if (op_ret) {
+                switch (op_errno) {
+                        case GF_DEPROBE_LOCALHOST:
+                                snprintf (errstr, len, "%s is localhost",
+                                hostname);
+                                break;
+
+                        case GF_DEPROBE_NOT_FRIEND:
+                                snprintf (errstr, len, "%s is not part of "
+                                          "cluster", hostname);
+                                break;
+
+                        case GF_DEPROBE_BRICK_EXIST:
+                                snprintf (errstr, len, "Brick(s) with the peer "
+                                          "%s exist in cluster", hostname);
+                                break;
+
+                        case GF_DEPROBE_FRIEND_DOWN:
+                                snprintf (errstr, len, "One of the peers is "
+                                          "probably down. Check with "
+                                          "'peer status'");
+                                break;
+
+                        case GF_DEPROBE_QUORUM_NOT_MET:
+                                snprintf (errstr, len, "Cluster quorum is not "
+                                          "met. Changing peers is not allowed "
+                                          "in this state");
+                                break;
+
+                        default:
+                                snprintf (errstr, len, "Detach returned with "
+                                          "unknown errno %d", op_errno);
+                                break;
+
+                }
+        }
+}
+
+
+int
+glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
+                                int32_t op_errno, char *op_errstr,
+                                char *hostname, dict_t *dict)
+{
+        gf_cli_rsp             rsp = {0,};
+        int32_t                ret = -1;
+        char              *cmd_str = NULL;
+        char          errstr[2048] = {0,};
+
+        GF_ASSERT (req);
+
+        (void) set_deprobe_error_str (op_ret, op_errno, op_errstr, errstr,
+                                      sizeof (errstr), hostname);
+
+        if (dict) {
+                ret = dict_get_str (dict, "cmd-str", &cmd_str);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to get "
+                                "command string");
+        }
+
+        rsp.op_ret = op_ret;
+        rsp.op_errno = op_errno;
+        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+
+        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
+                    (op_ret) ? "FAILED" : "SUCCESS",
+                    (errstr[0] != '\0') ? ":" : " ",
+                    (errstr[0] != '\0') ? errstr : " ");
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gf_cli_rsp);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to CLI, ret: %d",ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_list_friends (rpcsvc_request_t *req, dict_t *dict, int32_t flags)
+{
+        int32_t                 ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_peerinfo_t     *entry = NULL;
+        int32_t                 count = 0;
+        dict_t                  *friends = NULL;
+        gf1_cli_peer_list_rsp   rsp = {0,};
+        char                    my_uuid_str[64] = {0,};
+        char                    key[256] = {0,};
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        friends = dict_new ();
+        if (!friends) {
+                gf_log ("", GF_LOG_WARNING, "Out of Memory");
+                goto out;
+        }
+        if (!list_empty (&priv->peers)) {
+                list_for_each_entry (entry, &priv->peers, uuid_list) {
+                        count++;
+                        ret = glusterd_add_peer_detail_to_dict (entry,
+                                                                friends, count);
+                        if (ret)
+                                goto out;
+                }
+        }
+
+        if (flags == GF_CLI_LIST_POOL_NODES) {
+                count++;
+                snprintf (key, 256, "friend%d.uuid", count);
+                uuid_utoa_r (MY_UUID, my_uuid_str);
+                ret = dict_set_str (friends, key, my_uuid_str);
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.hostname", count);
+                ret = dict_set_str (friends, key, "localhost");
+                if (ret)
+                        goto out;
+
+                snprintf (key, 256, "friend%d.connected", count);
+                ret = dict_set_int32 (friends, key, 1);
+                if (ret)
+                        goto out;
+        }
+
+        ret = dict_set_int32 (friends, "count", count);
+        if (ret)
+                goto out;
+
+        ret = dict_allocate_and_serialize (friends, &rsp.friends.friends_val,
+                                           &rsp.friends.friends_len);
+
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+
+        if (friends)
+                dict_unref (friends);
+
+        rsp.op_ret = ret;
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
+        ret = 0;
+        GF_FREE (rsp.friends.friends_val);
+
+        return ret;
+}
+
+int32_t
+glusterd_get_volumes (rpcsvc_request_t *req, dict_t *dict, int32_t flags)
+{
+        int32_t                 ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_volinfo_t      *entry = NULL;
+        int32_t                 count = 0;
+        dict_t                  *volumes = NULL;
+        gf_cli_rsp              rsp = {0,};
+        char                    *volname = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        volumes = dict_new ();
+        if (!volumes) {
+                gf_log ("", GF_LOG_WARNING, "Out of Memory");
+                goto out;
+        }
+
+        if (list_empty (&priv->volumes)) {
+                ret = 0;
+                goto respond;
+        }
+
+        if (flags == GF_CLI_GET_VOLUME_ALL) {
+                list_for_each_entry (entry, &priv->volumes, vol_list) {
+                        ret = glusterd_add_volume_detail_to_dict (entry,
+                                                        volumes, count);
+                        if (ret)
+                                goto respond;
+
+                        count++;
+
+                }
+
+        } else if (flags == GF_CLI_GET_NEXT_VOLUME) {
+                ret = dict_get_str (dict, "volname", &volname);
+
+                if (ret) {
+                        if (priv->volumes.next) {
+                                entry = list_entry (priv->volumes.next,
+                                                    typeof (*entry),
+                                                    vol_list);
+                        }
+                } else {
+                        ret = glusterd_volinfo_find (volname, &entry);
+                        if (ret)
+                                goto respond;
+                        entry = list_entry (entry->vol_list.next,
+                                            typeof (*entry),
+                                            vol_list);
+                }
+
+                if (&entry->vol_list == &priv->volumes) {
+                       goto respond;
+                } else {
+                        ret = glusterd_add_volume_detail_to_dict (entry,
+                                                         volumes, count);
+                        if (ret)
+                                goto respond;
+
+                        count++;
+                }
+        } else if (flags == GF_CLI_GET_VOLUME) {
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto respond;
+
+                ret = glusterd_volinfo_find (volname, &entry);
+                if (ret)
+                        goto respond;
+
+                ret = glusterd_add_volume_detail_to_dict (entry,
+                                                 volumes, count);
+                if (ret)
+                        goto respond;
+
+                count++;
+        }
+
+respond:
+        ret = dict_set_int32 (volumes, "count", count);
+        if (ret)
+                goto out;
+        ret = dict_allocate_and_serialize (volumes, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        rsp.op_ret = ret;
+
+        rsp.op_errstr = "";
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_cli_rsp);
+        ret = 0;
+
+        if (volumes)
+                dict_unref (volumes);
+
+        GF_FREE (rsp.dict.dict_val);
+        return ret;
+}
+
+int
+__glusterd_handle_status_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret     = -1;
+        uint32_t                        cmd     = 0;
+        dict_t                         *dict    = NULL;
+        char                           *volname = 0;
+        gf_cli_req                      cli_req = {{0,}};
+        glusterd_op_t                   cli_op  = GD_OP_STATUS_VOLUME;
+        char                            err_str[2048] = {0,};
+        xlator_t                       *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len > 0) {
+                dict = dict_new();
+                if (!dict)
+                        goto out;
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len, &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize buffer");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                }
+
+        }
+
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        if (ret)
+                goto out;
+
+        if (!(cmd & GF_CLI_STATUS_ALL)) {
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str), "Unable to get "
+                                  "volume name");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_INFO,
+                        "Received status volume req for volume %s", volname);
+
+        }
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_STATUS_VOLUME, dict);
+
+out:
+
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        free (cli_req.dict.dict_val);
+
+        return ret;
+}
+
+int
+glusterd_handle_status_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_status_volume);
+}
+
+int
+__glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        glusterd_op_t                   cli_op = GD_OP_CLEARLOCKS_VOLUME;
+        char                            *volname = NULL;
+        dict_t                          *dict = NULL;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = -1;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to unserialize req-buffer to"
+                                " dictionary");
+                        snprintf (err_str, sizeof (err_str), "unable to decode "
+                                  "the command");
+                        goto out;
+                }
+
+        } else {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Empty cli request.");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Received clear-locks volume req "
+                "for volume %s", volname);
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_CLEARLOCKS_VOLUME, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        free (cli_req.dict.dict_val);
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_clearlocks_volume);
+}
+
+static int
+get_brickinfo_from_brickid (char *brickid, glusterd_brickinfo_t **brickinfo)
+{
+        glusterd_volinfo_t      *volinfo    = NULL;
+        char                    *volid_str  = NULL;
+        char                    *brick      = NULL;
+        char                    *brickid_dup = NULL;
+        uuid_t                  volid       = {0};
+        int                     ret         = -1;
+
+        brickid_dup = gf_strdup (brickid);
+        if (!brickid_dup)
+                goto out;
+
+        volid_str = brickid_dup;
+        brick = strchr (brickid_dup, ':');
+        *brick = '\0';
+        brick++;
+        if (!volid_str || !brick)
+                goto out;
+
+        uuid_parse (volid_str, volid);
+        ret = glusterd_volinfo_find_by_volume_id (volid, &volinfo);
+        if (ret) {
+                /* Check if it a snapshot volume */
+                ret = glusterd_snap_volinfo_find_by_volume_id (volid, &volinfo);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                      brickinfo);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        GF_FREE (brickid_dup);
+        return ret;
+}
+
+int
+__glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
+{
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *conf = NULL;
+        int                     ret = 0;
+        char                    *brickid = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+
+        brickid = mydata;
+        if (!brickid)
+                return 0;
+
+        ret = get_brickinfo_from_brickid (brickid, &brickinfo);
+        if (ret)
+                return 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        switch (event) {
+        case RPC_CLNT_CONNECT:
+                gf_log (this->name, GF_LOG_DEBUG, "Connected to %s:%s",
+                        brickinfo->hostname, brickinfo->path);
+                glusterd_set_brick_status (brickinfo, GF_BRICK_STARTED);
+                ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
+
+                break;
+
+        case RPC_CLNT_DISCONNECT:
+                if (GF_BRICK_STARTED == brickinfo->status)
+                        gf_log (this->name, GF_LOG_INFO, "Disconnected from "
+                                "%s:%s", brickinfo->hostname, brickinfo->path);
+
+                glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
+                if (rpc_clnt_is_disabled (rpc))
+                        GF_FREE (brickid);
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_TRACE,
+                        "got some other RPC event %d", event);
+                break;
+        }
+
+        return ret;
+}
+
+int
+glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                           rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event, data,
+                                           __glusterd_brick_rpc_notify);
+}
+
+int
+__glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                               rpc_clnt_event_t event, void *data)
+{
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *conf = NULL;
+        char                    *server = NULL;
+        int                     ret = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        server = mydata;
+        if (!server)
+                return 0;
+
+        switch (event) {
+        case RPC_CLNT_CONNECT:
+                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
+                (void) glusterd_nodesvc_set_online_status (server, _gf_true);
+                ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
+
+                break;
+
+        case RPC_CLNT_DISCONNECT:
+                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT");
+                (void) glusterd_nodesvc_set_online_status (server, _gf_false);
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_TRACE,
+                        "got some other RPC event %d", event);
+                break;
+        }
+
+        return ret;
+}
+
+int
+glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                             rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event, data,
+                                           __glusterd_nodesvc_rpc_notify);
+}
+
+int
+glusterd_friend_remove_notify (glusterd_peerctx_t *peerctx)
+{
+        int                             ret = -1;
+        glusterd_friend_sm_event_t      *new_event = NULL;
+        glusterd_peerinfo_t             *peerinfo = peerctx->peerinfo;
+        rpcsvc_request_t                *req = peerctx->args.req;
+        char                            *errstr = peerctx->errstr;
+        dict_t                          *dict = NULL;
+
+        GF_ASSERT (peerctx);
+
+        peerinfo = peerctx->peerinfo;
+        req = peerctx->args.req;
+        dict = peerctx->args.dict;
+        errstr = peerctx->errstr;
+
+        ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
+                                            &new_event);
+        if (!ret) {
+                if (!req) {
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "Unable to find the request for responding "
+                                "to User (%s)", peerinfo->hostname);
+                        goto out;
+                }
+
+                glusterd_xfer_cli_probe_resp (req, -1, ENOTCONN, errstr,
+                                              peerinfo->hostname,
+                                              peerinfo->port, dict);
+
+                new_event->peerinfo = peerinfo;
+                ret = glusterd_friend_sm_inject_event (new_event);
+
+        } else {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "Unable to create event for removing peer %s",
+                        peerinfo->hostname);
+        }
+
+out:
+        return ret;
+}
+
+int
+__glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                            rpc_clnt_event_t event, void *data)
+{
+        xlator_t             *this        = NULL;
+        glusterd_conf_t      *conf        = NULL;
+        int                   ret         = 0;
+        glusterd_peerinfo_t  *peerinfo    = NULL;
+        glusterd_peerctx_t   *peerctx     = NULL;
+        gf_boolean_t         quorum_action = _gf_false;
+        glusterd_volinfo_t   *volinfo     = NULL;
+
+        peerctx = mydata;
+        if (!peerctx)
+                return 0;
+
+        peerinfo = peerctx->peerinfo;
+        this = THIS;
+        conf = this->private;
+
+        switch (event) {
+        case RPC_CLNT_CONNECT:
+        {
+                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
+                peerinfo->connected = 1;
+                peerinfo->quorum_action = _gf_true;
+
+                ret = glusterd_peer_dump_version (this, rpc, peerctx);
+                if (ret)
+                        gf_log ("", GF_LOG_ERROR, "glusterd handshake failed");
+                break;
+        }
+
+        case RPC_CLNT_DISCONNECT:
+        {
+                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT %d",
+                        peerinfo->state.state);
+
+                if (peerinfo->connected) {
+                        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                                ret = glusterd_mgmt_v3_unlock (volinfo->volname,
+                                                               peerinfo->uuid,
+                                                               "vol");
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_TRACE,
+                                                "Lock not released for %s",
+                                                volinfo->volname);
+                        }
+
+                        ret = 0;
+                }
+
+                if ((peerinfo->quorum_contrib != QUORUM_DOWN) &&
+                    (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)) {
+                        peerinfo->quorum_contrib = QUORUM_DOWN;
+                        quorum_action = _gf_true;
+                        peerinfo->quorum_action = _gf_false;
+                }
+
+                /* Remove peer if it is not a friend and connection/handshake
+                *  fails, and notify cli. Happens only during probe.
+                */
+                if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT) {
+                        glusterd_friend_remove_notify (peerctx);
+                        goto out;
+                }
+
+                peerinfo->connected = 0;
+                break;
+        }
+        default:
+                gf_log (this->name, GF_LOG_TRACE,
+                        "got some other RPC event %d", event);
+                ret = 0;
+                break;
+        }
+
+out:
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        return ret;
+}
+
+int
+glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event, data,
+                                           __glusterd_peer_rpc_notify);
+}
+
+int
+glusterd_null (rpcsvc_request_t *req)
+{
+
+        return 0;
+}
+
+rpcsvc_actor_t gd_svc_mgmt_actors[] = {
+        [GLUSTERD_MGMT_NULL]           = { "NULL",           GLUSTERD_MGMT_NULL,           glusterd_null,                  NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_CLUSTER_LOCK]   = { "CLUSTER_LOCK",   GLUSTERD_MGMT_CLUSTER_LOCK,   glusterd_handle_cluster_lock,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = { "CLUSTER_UNLOCK", GLUSTERD_MGMT_CLUSTER_UNLOCK, glusterd_handle_cluster_unlock, NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_STAGE_OP]       = { "STAGE_OP",       GLUSTERD_MGMT_STAGE_OP,       glusterd_handle_stage_op,       NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_COMMIT_OP]      = { "COMMIT_OP",      GLUSTERD_MGMT_COMMIT_OP,      glusterd_handle_commit_op,      NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_mgmt_prog = {
+        .progname  = "GlusterD svc mgmt",
+        .prognum   = GD_MGMT_PROGRAM,
+        .progver   = GD_MGMT_VERSION,
+        .numactors = GLUSTERD_MGMT_MAXVALUE,
+        .actors    = gd_svc_mgmt_actors,
+	.synctask  = _gf_true,
+};
+
+rpcsvc_actor_t gd_svc_peer_actors[] = {
+        [GLUSTERD_FRIEND_NULL]    = { "NULL",          GLUSTERD_MGMT_NULL,     glusterd_null,                         NULL, 0, DRC_NA},
+        [GLUSTERD_PROBE_QUERY]    = { "PROBE_QUERY",   GLUSTERD_PROBE_QUERY,   glusterd_handle_probe_query,           NULL, 0, DRC_NA},
+        [GLUSTERD_FRIEND_ADD]     = { "FRIEND_ADD",    GLUSTERD_FRIEND_ADD,    glusterd_handle_incoming_friend_req,   NULL, 0, DRC_NA},
+        [GLUSTERD_FRIEND_REMOVE]  = { "FRIEND_REMOVE", GLUSTERD_FRIEND_REMOVE, glusterd_handle_incoming_unfriend_req, NULL, 0, DRC_NA},
+        [GLUSTERD_FRIEND_UPDATE]  = { "FRIEND_UPDATE", GLUSTERD_FRIEND_UPDATE, glusterd_handle_friend_update,         NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_peer_prog = {
+        .progname  = "GlusterD svc peer",
+        .prognum   = GD_FRIEND_PROGRAM,
+        .progver   = GD_FRIEND_VERSION,
+        .numactors = GLUSTERD_FRIEND_MAXVALUE,
+        .actors    = gd_svc_peer_actors,
+	.synctask  = _gf_false,
+};
+
+
+
+rpcsvc_actor_t gd_svc_cli_actors[] = {
+        [GLUSTER_CLI_PROBE]              = { "CLI_PROBE",         GLUSTER_CLI_PROBE,            glusterd_handle_cli_probe,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_CREATE_VOLUME]      = { "CLI_CREATE_VOLUME", GLUSTER_CLI_CREATE_VOLUME,    glusterd_handle_create_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_DEFRAG_VOLUME]      = { "CLI_DEFRAG_VOLUME", GLUSTER_CLI_DEFRAG_VOLUME,    glusterd_handle_defrag_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_DEPROBE]            = { "FRIEND_REMOVE",     GLUSTER_CLI_DEPROBE,          glusterd_handle_cli_deprobe,           NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_RESET]         = { "UUID_RESET",        GLUSTER_CLI_UUID_RESET,       glusterd_handle_cli_uuid_reset,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_START_VOLUME]       = { "START_VOLUME",      GLUSTER_CLI_START_VOLUME,     glusterd_handle_cli_start_volume,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_STOP_VOLUME]        = { "STOP_VOLUME",       GLUSTER_CLI_STOP_VOLUME,      glusterd_handle_cli_stop_volume,       NULL, 0, DRC_NA},
+        [GLUSTER_CLI_DELETE_VOLUME]      = { "DELETE_VOLUME",     GLUSTER_CLI_DELETE_VOLUME,    glusterd_handle_cli_delete_volume,     NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_ADD_BRICK]          = { "ADD_BRICK",         GLUSTER_CLI_ADD_BRICK,        glusterd_handle_add_brick,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_REPLACE_BRICK]      = { "REPLACE_BRICK",     GLUSTER_CLI_REPLACE_BRICK,    glusterd_handle_replace_brick,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_REMOVE_BRICK]       = { "REMOVE_BRICK",      GLUSTER_CLI_REMOVE_BRICK,     glusterd_handle_remove_brick,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LOG_ROTATE]         = { "LOG FILENAME",      GLUSTER_CLI_LOG_ROTATE,       glusterd_handle_log_rotate,            NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SET_VOLUME]         = { "SET_VOLUME",        GLUSTER_CLI_SET_VOLUME,       glusterd_handle_set_volume,            NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SYNC_VOLUME]        = { "SYNC_VOLUME",       GLUSTER_CLI_SYNC_VOLUME,      glusterd_handle_sync_volume,           NULL, 0, DRC_NA},
+        [GLUSTER_CLI_RESET_VOLUME]       = { "RESET_VOLUME",      GLUSTER_CLI_RESET_VOLUME,     glusterd_handle_reset_volume,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_FSM_LOG]            = { "FSM_LOG",           GLUSTER_CLI_FSM_LOG,          glusterd_handle_fsm_log,               NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GSYNC_SET]          = { "GSYNC_SET",         GLUSTER_CLI_GSYNC_SET,        glusterd_handle_gsync_set,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_PROFILE_VOLUME]     = { "STATS_VOLUME",      GLUSTER_CLI_PROFILE_VOLUME,   glusterd_handle_cli_profile_volume,    NULL, 0, DRC_NA},
+        [GLUSTER_CLI_QUOTA]              = { "QUOTA",             GLUSTER_CLI_QUOTA,            glusterd_handle_quota,                 NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_MOUNT]              = { "MOUNT",             GLUSTER_CLI_MOUNT,            glusterd_handle_mount,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_UMOUNT]             = { "UMOUNT",            GLUSTER_CLI_UMOUNT,           glusterd_handle_umount,                NULL, 1, DRC_NA},
+        [GLUSTER_CLI_HEAL_VOLUME]        = { "HEAL_VOLUME",       GLUSTER_CLI_HEAL_VOLUME,      glusterd_handle_cli_heal_volume,       NULL, 0, DRC_NA},
+        [GLUSTER_CLI_STATEDUMP_VOLUME]   = {"STATEDUMP_VOLUME",   GLUSTER_CLI_STATEDUMP_VOLUME, glusterd_handle_cli_statedump_volume,  NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
+        [GLUSTER_CLI_CLRLOCKS_VOLUME]    = {"CLEARLOCKS_VOLUME",  GLUSTER_CLI_CLRLOCKS_VOLUME,  glusterd_handle_cli_clearlocks_volume, NULL, 0, DRC_NA},
+        [GLUSTER_CLI_COPY_FILE]          = {"COPY_FILE",          GLUSTER_CLI_COPY_FILE,        glusterd_handle_copy_file,             NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SYS_EXEC]           = {"SYS_EXEC",           GLUSTER_CLI_SYS_EXEC,         glusterd_handle_sys_exec,              NULL, 0, DRC_NA},
+        [GLUSTER_CLI_SNAP]               = {"SNAP",               GLUSTER_CLI_SNAP,             glusterd_handle_snapshot,              NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_cli_prog = {
+        .progname  = "GlusterD svc cli",
+        .prognum   = GLUSTER_CLI_PROGRAM,
+        .progver   = GLUSTER_CLI_VERSION,
+        .numactors = GLUSTER_CLI_MAXVALUE,
+        .actors    = gd_svc_cli_actors,
+	.synctask  = _gf_true,
+};
+
+/* This is a minimal RPC prog, which contains only the readonly RPC procs from
+ * the cli rpcsvc
+ */
+rpcsvc_actor_t gd_svc_cli_actors_ro[] = {
+        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
+        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
+        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
+        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
+        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_cli_prog_ro = {
+        .progname  = "GlusterD svc cli read-only",
+        .prognum   = GLUSTER_CLI_PROGRAM,
+        .progver   = GLUSTER_CLI_VERSION,
+        .numactors = GLUSTER_CLI_MAXVALUE,
+        .actors    = gd_svc_cli_actors_ro,
+	.synctask  = _gf_true,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
new file mode 100644
index 000000000..0f0357c4c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -0,0 +1,1366 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "glusterfs.h"
+#include "compat-errno.h"
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+
+#include "glusterfs3.h"
+#include "protocol-common.h"
+#include "rpcsvc.h"
+#include "rpc-common-xdr.h"
+
+extern struct rpc_clnt_program gd_peer_prog;
+extern struct rpc_clnt_program gd_mgmt_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+
+
+#define TRUSTED_PREFIX         "trusted-"
+
+typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *data);
+
+static int
+get_snap_volname_and_volinfo (const char *volpath, char **volname,
+                              glusterd_volinfo_t **volinfo)
+{
+        int              ret            = -1;
+        char            *save_ptr       = NULL;
+        char            *str_token      = NULL;
+        char            *snapname       = NULL;
+        char            *volname_token  = NULL;
+        char            *vol            = NULL;
+        glusterd_snap_t *snap           = NULL;
+        xlator_t        *this           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (volpath);
+        GF_ASSERT (volinfo);
+
+        str_token = gf_strdup (volpath);
+        if (NULL == str_token) {
+                goto out;
+        }
+
+        /* Input volname will have below formats:
+         * /snaps/<snapname>/<volname>.<hostname>
+         * or
+         * /snaps/<snapname>/<parent-volname>
+         * We need to extract snapname and parent_volname */
+
+        /*split string by "/" */
+        strtok_r (str_token, "/",  &save_ptr);
+        snapname  = strtok_r(NULL, "/", &save_ptr);
+        if (!snapname) {
+                gf_log(this->name, GF_LOG_ERROR, "Invalid path: %s", volpath);
+                goto out;
+        }
+
+        volname_token = strtok_r(NULL, "/", &save_ptr);
+        if (!volname_token) {
+                gf_log(this->name, GF_LOG_ERROR, "Invalid path: %s", volpath);
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                gf_log(this->name, GF_LOG_ERROR, "Failed to "
+                                "fetch snap %s", snapname);
+                goto out;
+        }
+
+        /* Find if its a parent volume name or snap volume
+         * name. This function will succeed if volname_token
+         * is a parent volname
+         */
+        ret = glusterd_volinfo_find (volname_token, volinfo);
+        if (ret) {
+                *volname = gf_strdup (volname_token);
+                if (NULL == *volname) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_snap_volinfo_find (volname_token, snap,
+                                                  volinfo);
+                if (ret) {
+                        /* Split the volume name */
+                        vol = strtok_r (volname_token, ".", &save_ptr);
+                        if (!vol) {
+                                gf_log(this->name, GF_LOG_ERROR, "Invalid "
+                                                "volname (%s)", volname_token);
+                                goto out;
+                        }
+
+                        ret = glusterd_snap_volinfo_find (vol, snap, volinfo);
+                        if (ret) {
+                                gf_log(this->name, GF_LOG_ERROR, "Failed to "
+                                       "fetch snap volume from volname (%s)",
+                                       vol);
+                                goto out;
+                        }
+                }
+        } else {
+                /*volname_token is parent volname*/
+                ret = glusterd_snap_volinfo_find_from_parent_volname (
+                                volname_token, snap, volinfo);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_ERROR, "Failed to "
+                                        "fetch snap volume from parent "
+                                        "volname (%s)", volname_token);
+                        goto out;
+                }
+
+                /* Since volname_token is a parent volname we should
+                 * get the snap volname here*/
+                *volname = gf_strdup ((*volinfo)->volname);
+                if (NULL == *volname) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+out:
+        if (ret && NULL != *volname) {
+                GF_FREE (*volname);
+                *volname = NULL;
+        }
+        return ret;
+}
+
+static size_t
+build_volfile_path (const char *volname, char *path,
+                    size_t path_len, char *trusted_str)
+{
+        struct stat              stbuf                  = {0,};
+        int32_t                  ret                    = -1;
+        glusterd_conf_t         *priv                   = NULL;
+        char                    *vol                    = NULL;
+        char                    *dup_volname            = NULL;
+        char                    *free_ptr               = NULL;
+        char                    *save_ptr               = NULL;
+        char                    *str_token              = NULL;
+        glusterd_volinfo_t      *volinfo                = NULL;
+        char                    *server                 = NULL;
+        const char              *volname_ptr            = NULL;
+        char                     path_prefix [PATH_MAX] = {0,};
+        xlator_t                *this                   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volname);
+        GF_ASSERT (path);
+
+        if (strstr (volname, "gluster/")) {
+                server = strchr (volname, '/') + 1;
+                glusterd_get_nodesvc_volfile (server, priv->workdir,
+                                              path, path_len);
+                ret = 1;
+                goto out;
+        } else if ((str_token = strstr (volname, "/snaps/"))) {
+                ret = get_snap_volname_and_volinfo (str_token, &dup_volname,
+                                                    &volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get snap"
+                                " volinfo from path (%s)", volname);
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (path_prefix, sizeof (path_prefix), "%s/snaps/%s",
+                          priv->workdir, volinfo->snapshot->snapname);
+
+                free_ptr = dup_volname;
+                volname_ptr = dup_volname;
+                goto gotvolinfo;
+        } else if (volname[0] != '/') {
+                /* Normal behavior */
+                dup_volname = gf_strdup (volname);
+        } else {
+                /* Bringing in NFS like behavior for mount command,    */
+                /* With this, one can mount a volume with below cmd    */
+                /* bash# mount -t glusterfs server:/volume /mnt/pnt    */
+                dup_volname = gf_strdup (&volname[1]);
+        }
+
+        if (!dup_volname) {
+                gf_log(THIS->name, GF_LOG_ERROR, "strdup failed");
+                ret = -1;
+                goto out;
+        }
+        free_ptr = dup_volname;
+        volname_ptr = volname;
+
+        snprintf (path_prefix, sizeof (path_prefix), "%s/vols",
+                  priv->workdir);
+
+        ret = glusterd_volinfo_find (dup_volname, &volinfo);
+
+        if (ret) {
+                /* Split the volume name */
+                vol = strtok_r (dup_volname, ".", &save_ptr);
+                if (!vol)
+                        goto out;
+
+                ret = glusterd_volinfo_find (vol, &volinfo);
+                if (ret)
+                        goto out;
+        }
+
+gotvolinfo:
+        if (!glusterd_auth_get_username (volinfo))
+                trusted_str = NULL;
+
+        ret = snprintf (path, path_len, "%s/%s/%s.vol", path_prefix,
+                        volinfo->volname, volname_ptr);
+        if (ret == -1)
+                goto out;
+
+        ret = stat (path, &stbuf);
+
+        if ((ret == -1) && (errno == ENOENT)) {
+                snprintf (path, path_len, "%s/%s/%s%s-fuse.vol",
+                          path_prefix, volinfo->volname,
+                          (trusted_str ? trusted_str : ""),
+                          dup_volname);
+
+                ret = stat (path, &stbuf);
+        }
+
+        if ((ret == -1) && (errno == ENOENT)) {
+                snprintf (path, path_len, "%s/%s/%s-tcp.vol",
+                          path_prefix, volinfo->volname, volname_ptr);
+        }
+
+        ret = 1;
+out:
+        GF_FREE (free_ptr);
+        return ret;
+}
+
+/* Get and store op-versions of the clients sending the getspec request
+ * Clients of versions <= 3.3, don't send op-versions, their op-versions are
+ * defaulted to 1
+ */
+static int
+_get_client_op_versions (gf_getspec_req *args, peer_info_t *peerinfo)
+{
+        int    ret                   = 0;
+        int    client_max_op_version = 1;
+        int    client_min_op_version = 1;
+        dict_t *dict                 = NULL;
+
+        GF_ASSERT (args);
+        GF_ASSERT (peerinfo);
+
+        if (args->xdata.xdata_len) {
+                dict = dict_new ();
+                if (!dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_unserialize (args->xdata.xdata_val,
+                                        args->xdata.xdata_len, &dict);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Failed to unserialize request dictionary");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "min-op-version",
+                                      &client_min_op_version);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Failed to get client-min-op-version");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "max-op-version",
+                                      &client_max_op_version);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Failed to get client-max-op-version");
+                        goto out;
+                }
+        }
+
+        peerinfo->max_op_version = client_max_op_version;
+        peerinfo->min_op_version = client_min_op_version;
+
+out:
+        return ret;
+}
+
+/* Checks if the client supports the volume, ie. client can understand all the
+ * options in the volfile
+ */
+static gf_boolean_t
+_client_supports_volume (peer_info_t *peerinfo, int32_t *op_errno)
+{
+        gf_boolean_t       ret       = _gf_true;
+        glusterd_volinfo_t *volinfo  = NULL;
+
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (op_errno);
+
+
+        /* Only check when the volfile being requested is a volume. Not finding
+         * a volinfo implies that the volfile requested for is not of a gluster
+         * volume. A non volume volfile is requested by the local gluster
+         * services like shd and nfs-server. These need not be checked as they
+         * will be running at the same op-version as glusterd and will be able
+         * to support all the features
+         */
+        if ((glusterd_volinfo_find (peerinfo->volname, &volinfo) == 0) &&
+            ((peerinfo->min_op_version > volinfo->client_op_version) ||
+             (peerinfo->max_op_version < volinfo->client_op_version))) {
+                ret = _gf_false;
+                *op_errno = ENOTSUP;
+                gf_log ("glusterd", GF_LOG_INFO,
+                        "Client %s (%d -> %d) doesn't support required "
+                        "op-version (%d). Rejecting volfile request.",
+                        peerinfo->identifier, peerinfo->min_op_version,
+                        peerinfo->max_op_version, volinfo->client_op_version);
+        }
+
+        return ret;
+}
+
+int
+__server_getspec (rpcsvc_request_t *req)
+{
+        int32_t               ret                    = -1;
+        int32_t               op_errno               = 0;
+        int32_t               spec_fd                = -1;
+        size_t                file_len               = 0;
+        char                  filename[PATH_MAX]  = {0,};
+        struct stat           stbuf                  = {0,};
+        char                 *volume                 = NULL;
+        char                 *tmp                    = NULL;
+        int                   cookie                 = 0;
+        rpc_transport_t      *trans                  = NULL;
+        gf_getspec_req        args                   = {0,};
+        gf_getspec_rsp        rsp                    = {0,};
+        char                  addrstr[RPCSVC_PEER_STRLEN] = {0};
+        peer_info_t          *peerinfo               = NULL;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_getspec_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        peerinfo = &req->trans->peerinfo;
+
+        volume = args.key;
+        /* Need to strip leading '/' from volnames. This was introduced to
+         * support nfs style mount parameters for native gluster mount
+         */
+        if (volume[0] == '/')
+                strncpy (peerinfo->volname, &volume[1], strlen(&volume[1]));
+        else
+                strncpy (peerinfo->volname, volume, strlen(volume));
+
+        ret = _get_client_op_versions (&args, peerinfo);
+        if (ret)
+                goto fail;
+
+        if (!_client_supports_volume (peerinfo, &op_errno)) {
+                ret = -1;
+                goto fail;
+        }
+
+        trans = req->trans;
+        /* addrstr will be empty for cli socket connections */
+        ret = rpcsvc_transport_peername (trans, (char *)&addrstr,
+                                         sizeof (addrstr));
+        if (ret)
+                goto fail;
+
+        tmp  = strrchr (addrstr, ':');
+        if (tmp)
+                *tmp = '\0';
+
+        /* The trusted volfiles are given to the glusterd owned process like NFS
+         * server, self-heal daemon etc., so that they are not inadvertently
+         * blocked by a auth.{allow,reject} setting. The trusted volfile is not
+         * meant for external users.
+         */
+        if (strlen (addrstr) && gf_is_local_addr (addrstr)) {
+
+                ret = build_volfile_path (volume, filename,
+                                          sizeof (filename),
+                                          TRUSTED_PREFIX);
+        } else {
+                ret = build_volfile_path (volume, filename,
+                                          sizeof (filename), NULL);
+        }
+
+        if (ret > 0) {
+                /* to allocate the proper buffer to hold the file data */
+                ret = stat (filename, &stbuf);
+                if (ret < 0){
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Unable to stat %s (%s)",
+                                filename, strerror (errno));
+                        goto fail;
+                }
+
+                spec_fd = open (filename, O_RDONLY);
+                if (spec_fd < 0) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Unable to open %s (%s)",
+                                filename, strerror (errno));
+                        goto fail;
+                }
+                ret = file_len = stbuf.st_size;
+        } else {
+                op_errno = ENOENT;
+        }
+
+        if (file_len) {
+                rsp.spec = CALLOC (file_len+1, sizeof (char));
+                if (!rsp.spec) {
+                        ret = -1;
+                        op_errno = ENOMEM;
+                        goto fail;
+                }
+                ret = read (spec_fd, rsp.spec, file_len);
+
+                close (spec_fd);
+        }
+
+        /* convert to XDR */
+fail:
+        rsp.op_ret   = ret;
+
+        if (op_errno)
+                rsp.op_errno = gf_errno_to_error (op_errno);
+        if (cookie)
+                rsp.op_errno = cookie;
+
+        if (!rsp.spec)
+                rsp.spec = strdup ("");
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_getspec_rsp);
+        free (args.key);//malloced by xdr
+        free (rsp.spec);
+
+        return 0;
+}
+
+int
+server_getspec (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __server_getspec);
+}
+
+int32_t
+__server_event_notify (rpcsvc_request_t *req)
+{
+        int32_t                 ret             = -1;
+        int32_t                 op_errno        =  0;
+        gf_event_notify_req     args            = {0,};
+        gf_event_notify_rsp     rsp             = {0,};
+        dict_t                 *dict            = NULL;
+        gf_boolean_t            need_rsp        = _gf_true;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_event_notify_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        if (args.dict.dict_len) {
+                dict = dict_new ();
+                if (!dict)
+                        return ret;
+                ret = dict_unserialize (args.dict.dict_val,
+                                        args.dict.dict_len, &dict);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to unserialize req");
+                        goto fail;
+                }
+        }
+
+        switch (args.op) {
+        case GF_EN_DEFRAG_STATUS:
+                gf_log ("", GF_LOG_INFO,
+                        "received defrag status updated");
+                if (dict) {
+                        glusterd_defrag_event_notify_handle (dict);
+                        need_rsp = _gf_false;
+                }
+                break;
+        default:
+                gf_log ("", GF_LOG_ERROR, "Unknown op received in event "
+                        "notify");
+                ret = -1;
+                break;
+        }
+
+fail:
+        rsp.op_ret   = ret;
+
+        if (op_errno)
+                rsp.op_errno = gf_errno_to_error (op_errno);
+
+        if (need_rsp)
+                glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                       (xdrproc_t)xdr_gf_event_notify_rsp);
+        if (dict)
+                dict_unref (dict);
+        free (args.dict.dict_val);//malloced by xdr
+
+        return 0;
+}
+
+int32_t
+server_event_notify (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __server_event_notify);
+}
+
+int
+gd_validate_cluster_op_version (xlator_t *this, int cluster_op_version,
+                                char *peerid)
+{
+        int              ret  = -1;
+        glusterd_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (cluster_op_version > GD_OP_VERSION_MAX) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "operating version %d is more than the maximum "
+                        "supported (%d) on the machine (as per peer request "
+                        "from %s)", cluster_op_version, GD_OP_VERSION_MAX,
+                        peerid);
+                goto out;
+        }
+
+        /* The peer can only reduce its op-version when it doesn't have any
+         * volumes. Reducing op-version when it already contains volumes can
+         * lead to inconsistencies in the cluster
+         */
+        if ((cluster_op_version < conf->op_version) &&
+            !list_empty (&conf->volumes)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot reduce operating version to %d from current "
+                        "version %d as volumes exist (as per peer request from "
+                        "%s)", cluster_op_version, conf->op_version, peerid);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+__glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+{
+        dict_t            *dict            = NULL;
+        xlator_t          *this            = NULL;
+        glusterd_conf_t   *conf            = NULL;
+        int                ret             = -1;
+        int                op_errno        = EINVAL;
+        gf_mgmt_hndsk_req  args            = {{0,},};
+        gf_mgmt_hndsk_rsp  rsp             = {0,};
+
+        this = THIS;
+        conf = this->private;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_int32 (dict, GD_OP_VERSION_KEY, conf->op_version);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set operating version");
+                rsp.op_ret = ret;
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, GD_MIN_OP_VERSION_KEY, GD_OP_VERSION_MIN);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s", GD_MIN_OP_VERSION_KEY);
+                rsp.op_ret = ret;
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, GD_MAX_OP_VERSION_KEY, GD_OP_VERSION_MAX);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s", GD_MAX_OP_VERSION_KEY);
+                rsp.op_ret = ret;
+                goto out;
+        }
+
+        ret = 0;
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, dict, (&rsp.hndsk.hndsk_val),
+                                    rsp.hndsk.hndsk_len, op_errno, out);
+out:
+
+        rsp.op_ret = ret;
+        rsp.op_errno = op_errno;
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+
+        ret = 0;
+
+        if (dict)
+                dict_unref (dict);
+
+        if (args.hndsk.hndsk_val)
+                free (args.hndsk.hndsk_val);
+
+        if (rsp.hndsk.hndsk_val)
+                GF_FREE (rsp.hndsk.hndsk_val);
+
+        return ret;
+}
+
+int
+glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_mgmt_hndsk_versions);
+}
+
+int
+__glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+{
+        dict_t            *clnt_dict       = NULL;
+        xlator_t          *this            = NULL;
+        glusterd_conf_t   *conf            = NULL;
+        int                ret             = -1;
+        int                op_errno        = EINVAL;
+        int                peer_op_version = 0;
+        gf_mgmt_hndsk_req  args            = {{0,},};
+        gf_mgmt_hndsk_rsp  rsp             = {0,};
+
+        this = THIS;
+        conf = this->private;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, clnt_dict, args.hndsk.hndsk_val,
+                                      (args.hndsk.hndsk_len), ret, op_errno,
+                                      out);
+
+        ret = dict_get_int32 (clnt_dict, GD_OP_VERSION_KEY, &peer_op_version);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to get the op-version key peer=%s",
+                        req->trans->peerinfo.identifier);
+                goto out;
+        }
+
+        ret = gd_validate_cluster_op_version (this, peer_op_version,
+                                              req->trans->peerinfo.identifier);
+        if (ret)
+                goto out;
+
+
+        /* As this is ACK from the Cluster for the versions supported,
+           can set the op-version of 'this' glusterd to the one
+           received. */
+        gf_log (this->name, GF_LOG_INFO, "using the op-version %d",
+                peer_op_version);
+        conf->op_version = peer_op_version;
+        ret = glusterd_store_global_info (this);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store op-version");
+
+out:
+        rsp.op_ret = ret;
+        rsp.op_errno = op_errno;
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+
+        ret = 0;
+
+        if (clnt_dict)
+                dict_unref (clnt_dict);
+
+        if (args.hndsk.hndsk_val)
+                free (args.hndsk.hndsk_val);
+
+        return ret;
+}
+
+int
+glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_mgmt_hndsk_versions_ack);
+}
+
+rpcsvc_actor_t gluster_handshake_actors[] = {
+        [GF_HNDSK_NULL]         = {"NULL",        GF_HNDSK_NULL,         NULL,                NULL, 0, DRC_NA},
+        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
+        [GF_HNDSK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_HNDSK_EVENT_NOTIFY, server_event_notify, NULL, 0, DRC_NA},
+};
+
+
+struct rpcsvc_program gluster_handshake_prog = {
+        .progname  = "Gluster Handshake",
+        .prognum   = GLUSTER_HNDSK_PROGRAM,
+        .progver   = GLUSTER_HNDSK_VERSION,
+        .actors    = gluster_handshake_actors,
+        .numactors = GF_HNDSK_MAXVALUE,
+};
+
+/* A minimal RPC program just for the cli getspec command */
+rpcsvc_actor_t gluster_cli_getspec_actors[] = {
+        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gluster_cli_getspec_prog = {
+        .progname  = "Gluster Handshake (CLI Getspec)",
+        .prognum   = GLUSTER_HNDSK_PROGRAM,
+        .progver   = GLUSTER_HNDSK_VERSION,
+        .actors    = gluster_cli_getspec_actors,
+        .numactors = GF_HNDSK_MAXVALUE,
+};
+
+
+char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = {
+        [GF_DUMP_NULL] = "NULL",
+        [GF_DUMP_DUMP] = "DUMP",
+};
+
+rpc_clnt_prog_t glusterd_dump_prog = {
+        .progname       = "GLUSTERD-DUMP",
+        .prognum        = GLUSTER_DUMP_PROGRAM,
+        .progver        = GLUSTER_DUMP_VERSION,
+        .procnames      = glusterd_dump_proc,
+};
+
+
+rpcsvc_actor_t glusterd_mgmt_hndsk_actors[] = {
+        [GD_MGMT_HNDSK_NULL]            = {"NULL", GD_MGMT_HNDSK_NULL, NULL,
+                                           NULL, 0},
+        [GD_MGMT_HNDSK_VERSIONS]        = {"MGMT-VERS", GD_MGMT_HNDSK_VERSIONS,
+                                           glusterd_mgmt_hndsk_versions, NULL,
+                                           0},
+        [GD_MGMT_HNDSK_VERSIONS_ACK]    = {"MGMT-VERS-ACK",
+                                           GD_MGMT_HNDSK_VERSIONS_ACK,
+                                           glusterd_mgmt_hndsk_versions_ack,
+                                           NULL, 0},
+};
+
+struct rpcsvc_program glusterd_mgmt_hndsk_prog = {
+        .progname       = "Gluster MGMT Handshake",
+        .prognum        = GD_MGMT_HNDSK_PROGRAM,
+        .progver        = GD_MGMT_HNDSK_VERSION,
+        .actors         = glusterd_mgmt_hndsk_actors,
+        .numactors      = GD_MGMT_HNDSK_MAXVALUE,
+};
+
+char *glusterd_mgmt_hndsk_proc[GD_MGMT_HNDSK_MAXVALUE] = {
+        [GD_MGMT_HNDSK_NULL]          = "NULL",
+        [GD_MGMT_HNDSK_VERSIONS] = "MGMT-VERS",
+        [GD_MGMT_HNDSK_VERSIONS_ACK] = "MGMT-VERS-ACK",
+};
+
+rpc_clnt_prog_t gd_clnt_mgmt_hndsk_prog = {
+        .progname  = "Gluster MGMT Handshake",
+        .prognum   = GD_MGMT_HNDSK_PROGRAM,
+        .progver   = GD_MGMT_HNDSK_VERSION,
+        .procnames = glusterd_mgmt_hndsk_proc,
+};
+
+
+static int
+glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
+{
+        GF_ASSERT (peerctx);
+
+        glusterd_friend_sm_event_t      *event = NULL;
+        glusterd_probe_ctx_t            *ctx = NULL;
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+
+
+        ret = glusterd_friend_sm_new_event
+                        (GD_FRIEND_EVENT_CONNECTED, &event);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get new event");
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
+
+        if (!ctx) {
+                ret = -1;
+                gf_log ("", GF_LOG_ERROR, "Memory not available");
+                goto out;
+        }
+
+        peerinfo = peerctx->peerinfo;
+        ctx->hostname = gf_strdup (peerinfo->hostname);
+        ctx->port = peerinfo->port;
+        ctx->req = peerctx->args.req;
+        ctx->dict = peerctx->args.dict;
+
+        event->peerinfo = peerinfo;
+        event->ctx = ctx;
+
+        ret = glusterd_friend_sm_inject_event (event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject "
+                        "EVENT_CONNECTED ret = %d", ret);
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+
+int
+gd_validate_peer_op_version (xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                             dict_t *dict, char **errstr)
+{
+        int              ret  = -1;
+        glusterd_conf_t *conf = NULL;
+        int32_t          peer_op_version = 0;
+        int32_t          peer_min_op_version = 0;
+        int32_t          peer_max_op_version = 0;
+
+        if (!dict && !this && !peerinfo)
+                goto out;
+
+        conf = this->private;
+
+        ret = dict_get_int32 (dict, GD_OP_VERSION_KEY, &peer_op_version);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, GD_MAX_OP_VERSION_KEY,
+                              &peer_max_op_version);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, GD_MIN_OP_VERSION_KEY,
+                              &peer_min_op_version);
+        if (ret)
+                goto out;
+
+        ret = -1;
+        /* Check if peer can support our op_version */
+        if ((peer_max_op_version < conf->op_version) ||
+            (peer_min_op_version > conf->op_version)) {
+                ret = gf_asprintf (errstr, "Peer %s does not support required "
+                                   "op-version", peerinfo->hostname);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name , GF_LOG_DEBUG, "Peer %s %s", peerinfo->hostname,
+                ((ret < 0) ? "rejected" : "accepted"));
+        return ret;
+}
+
+int
+__glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+        int                  ret      = -1;
+        int                  op_errno = EINVAL;
+        gf_mgmt_hndsk_rsp    rsp      = {0,};
+        xlator_t            *this     = NULL;
+        call_frame_t        *frame    = NULL;
+        glusterd_peerinfo_t *peerinfo = NULL;
+        glusterd_peerctx_t  *peerctx  = NULL;
+        char                msg[1024] = {0,};
+
+        this = THIS;
+        frame = myframe;
+        peerctx = frame->local;
+        peerinfo = peerctx->peerinfo;
+
+        if (-1 == req->rpc_status) {
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+        if (ret < 0) {
+                snprintf (msg, sizeof (msg), "Failed to decode XDR");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        op_errno = rsp.op_errno;
+        if (-1 == rsp.op_ret) {
+                ret = -1;
+                snprintf (msg, sizeof (msg),
+                          "Failed to get handshake ack from remote server");
+                gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        /* TODO: this is hardcoded as of now, but I don't forsee any problems
+         * with this as long as we are properly handshaking operating versions
+         */
+        peerinfo->mgmt = &gd_mgmt_prog;
+        peerinfo->peer = &gd_peer_prog;
+        peerinfo->mgmt_v3 = &gd_mgmt_v3_prog;
+
+        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
+
+        if (GD_MODE_ON == peerctx->args.mode) {
+                ret = glusterd_event_connected_inject (peerctx);
+                peerctx->args.req = NULL;
+        } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+                peerctx->args.mode = GD_MODE_ON;
+        } else {
+                gf_log (this->name, GF_LOG_WARNING, "unknown mode %d",
+                        peerctx->args.mode);
+        }
+
+        glusterd_friend_sm ();
+
+        ret = 0;
+out:
+
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        if (ret != 0)
+                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+
+        if (rsp.hndsk.hndsk_val)
+                free (rsp.hndsk.hndsk_val);
+
+        return 0;
+}
+
+int
+glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_mgmt_hndsk_version_ack_cbk);
+}
+
+int
+__glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int                  ret       = -1;
+        int                  op_errno  = EINVAL;
+        gf_mgmt_hndsk_rsp    rsp       = {0,};
+        gf_mgmt_hndsk_req    arg       = {{0,}};
+        xlator_t            *this      = NULL;
+        call_frame_t        *frame     = NULL;
+        glusterd_peerinfo_t *peerinfo  = NULL;
+        glusterd_peerctx_t  *peerctx   = NULL;
+        dict_t              *dict      = NULL;
+        dict_t              *rsp_dict  = NULL;
+        glusterd_conf_t     *conf      = NULL;
+        char                 msg[1024] = {0,};
+
+        this = THIS;
+        conf = this->private;
+        frame = myframe;
+        peerctx = frame->local;
+        peerinfo = peerctx->peerinfo;
+
+        if (-1 == req->rpc_status) {
+                ret = -1;
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+        if (ret < 0) {
+                snprintf (msg, sizeof (msg), "Failed to decode management "
+                          "handshake response");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_UNSERIALIZE (this, dict, rsp.hndsk.hndsk_val,
+                                      rsp.hndsk.hndsk_len, ret, op_errno,
+                                      out);
+
+        op_errno = rsp.op_errno;
+        if (-1 == rsp.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get the 'versions' from peer (%s)",
+                        req->conn->trans->peerinfo.identifier);
+                goto out;
+        }
+
+        /* Check if peer can be part of cluster */
+        ret = gd_validate_peer_op_version (this, peerinfo, dict,
+                                           &peerctx->errstr);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to validate the operating version of peer (%s)",
+                        peerinfo->hostname);
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict)
+                goto out;
+
+        ret = dict_set_int32 (rsp_dict, GD_OP_VERSION_KEY, conf->op_version);
+        if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to set operating version in dict");
+                goto out;
+        }
+
+        GF_PROTOCOL_DICT_SERIALIZE (this, rsp_dict, (&arg.hndsk.hndsk_val),
+                                    arg.hndsk.hndsk_len, op_errno, out);
+
+        ret = glusterd_submit_request (peerctx->peerinfo->rpc, &arg, frame,
+                                       &gd_clnt_mgmt_hndsk_prog,
+                                       GD_MGMT_HNDSK_VERSIONS_ACK, NULL, this,
+                                       glusterd_mgmt_hndsk_version_ack_cbk,
+                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+
+out:
+        if (ret) {
+                frame->local = NULL;
+                STACK_DESTROY (frame->root);
+                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+        }
+
+        if (rsp.hndsk.hndsk_val)
+                free (rsp.hndsk.hndsk_val);
+
+        if (arg.hndsk.hndsk_val)
+                GF_FREE (arg.hndsk.hndsk_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        return 0;
+}
+
+int
+glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_mgmt_hndsk_version_cbk);
+}
+
+int
+glusterd_mgmt_handshake (xlator_t *this, glusterd_peerctx_t *peerctx)
+{
+        call_frame_t        *frame    = NULL;
+        gf_mgmt_hndsk_req    req      = {{0,},};
+        int                  ret      = -1;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        frame->local = peerctx;
+
+        ret = glusterd_submit_request (peerctx->peerinfo->rpc, &req, frame,
+                                       &gd_clnt_mgmt_hndsk_prog,
+                                       GD_MGMT_HNDSK_VERSIONS, NULL, this,
+                                       glusterd_mgmt_hndsk_version_cbk,
+                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+        ret = 0;
+out:
+        if (ret && frame)
+                STACK_DESTROY (frame->root);
+
+        return ret;
+}
+
+int
+glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo,
+                                gf_prog_detail *prog)
+{
+        gf_prog_detail  *trav   = NULL;
+        int             ret     = -1;
+
+        if (!peerinfo || !prog)
+                goto out;
+
+        trav = prog;
+
+        while (trav) {
+                ret = -1;
+                if ((gd_mgmt_prog.prognum == trav->prognum) &&
+                    (gd_mgmt_prog.progver == trav->progver)) {
+                        peerinfo->mgmt = &gd_mgmt_prog;
+                        ret = 0;
+                }
+
+                if ((gd_peer_prog.prognum == trav->prognum) &&
+                    (gd_peer_prog.progver == trav->progver)) {
+                        peerinfo->peer = &gd_peer_prog;
+                        ret = 0;
+                }
+
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "%s (%"PRId64":%"PRId64") not supported",
+                                trav->progname, trav->prognum,
+                                trav->progver);
+                }
+
+                trav = trav->next;
+        }
+
+        if (peerinfo->mgmt) {
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->mgmt->progname, peerinfo->mgmt->prognum,
+                         peerinfo->mgmt->progver);
+        }
+
+        if (peerinfo->peer) {
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->peer->progname, peerinfo->peer->prognum,
+                         peerinfo->peer->progver);
+        }
+
+        if (peerinfo->mgmt_v3) {
+                 gf_log ("", GF_LOG_INFO,
+                         "Using Program %s, Num (%d), Version (%d)",
+                         peerinfo->mgmt_v3->progname, peerinfo->mgmt_v3->prognum,
+                         peerinfo->mgmt_v3->progver);
+        }
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+static gf_boolean_t
+_mgmt_hndsk_prog_present (gf_prog_detail *prog) {
+        gf_boolean_t    ret = _gf_false;
+        gf_prog_detail  *trav = NULL;
+
+        GF_ASSERT (prog);
+
+        trav = prog;
+
+        while (trav) {
+                if ((trav->prognum == GD_MGMT_HNDSK_PROGRAM) &&
+                    (trav->progver == GD_MGMT_HNDSK_VERSION)) {
+                        ret = _gf_true;
+                        goto out;
+                }
+                trav = trav->next;
+        }
+out:
+        return ret;
+}
+
+int
+__glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        int                  ret      = -1;
+        gf_dump_rsp          rsp      = {0,};
+        xlator_t            *this     = NULL;
+        gf_prog_detail      *trav     = NULL;
+        gf_prog_detail      *next     = NULL;
+        call_frame_t        *frame    = NULL;
+        glusterd_peerinfo_t *peerinfo = NULL;
+        glusterd_peerctx_t  *peerctx  = NULL;
+        glusterd_conf_t     *conf     = NULL;
+        char                msg[1024] = {0,};
+
+        this = THIS;
+        conf = this->private;
+        frame = myframe;
+        peerctx = frame->local;
+        peerinfo = peerctx->peerinfo;
+
+        if (-1 == req->rpc_status) {
+                snprintf (msg, sizeof (msg),
+                          "Error through RPC layer, retry again later");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
+        if (ret < 0) {
+                snprintf (msg, sizeof (msg), "Failed to decode XDR");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+        if (-1 == rsp.op_ret) {
+                snprintf (msg, sizeof (msg),
+                          "Failed to get the 'versions' from remote server");
+                gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg);
+                peerctx->errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        if (_mgmt_hndsk_prog_present (rsp.prog)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Proceeding to op-version handshake with peer %s",
+                        peerinfo->hostname);
+                ret = glusterd_mgmt_handshake (this, peerctx);
+                goto out;
+        } else if (conf->op_version > 1) {
+                ret = -1;
+                snprintf (msg, sizeof (msg),
+                          "Peer %s does not support required op-version",
+                          peerinfo->hostname);
+                peerctx->errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        /* Make sure we assign the proper program to peer */
+        ret = glusterd_set_clnt_mgmt_program (peerinfo, rsp.prog);
+        if (ret) {
+                gf_log ("", GF_LOG_WARNING, "failed to set the mgmt program");
+                goto out;
+        }
+
+        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
+
+        if (GD_MODE_ON == peerctx->args.mode) {
+                ret = glusterd_event_connected_inject (peerctx);
+                peerctx->args.req = NULL;
+        } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+                peerctx->args.mode = GD_MODE_ON;
+        } else {
+                gf_log ("", GF_LOG_WARNING, "unknown mode %d",
+                        peerctx->args.mode);
+        }
+
+        glusterd_friend_sm();
+        glusterd_op_sm();
+
+        ret = 0;
+
+out:
+
+        /* don't use GF_FREE, buffer was allocated by libc */
+        if (rsp.prog) {
+                trav = rsp.prog;
+                while (trav) {
+                        next = trav->next;
+                        free (trav->progname);
+                        free (trav);
+                        trav = next;
+                }
+        }
+
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        if (ret != 0)
+                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+
+        return 0;
+}
+
+
+int
+glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_peer_dump_version_cbk);
+}
+
+int
+glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
+                            glusterd_peerctx_t *peerctx)
+{
+        call_frame_t        *frame    = NULL;
+        gf_dump_req          req      = {0,};
+        int                  ret      = -1;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        frame->local = peerctx;
+
+        req.gfs_id = 0xcafe;
+
+        ret = glusterd_submit_request (peerctx->peerinfo->rpc, &req, frame,
+                                       &glusterd_dump_prog, GF_DUMP_DUMP,
+                                       NULL, this,
+                                       glusterd_peer_dump_version_cbk,
+                                       (xdrproc_t)xdr_gf_dump_req);
+out:
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.c b/xlators/mgmt/glusterd/src/glusterd-hooks.c
new file mode 100644
index 000000000..2b43a452e
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.c
@@ -0,0 +1,531 @@
+/*
+   Copyright (c) 2007-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "dict.h"
+#include "xlator.h"
+#include "logging.h"
+#include "run.h"
+#include "defaults.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "glusterd.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "glusterd-hooks.h"
+
+#include <fnmatch.h>
+
+#define EMPTY ""
+char glusterd_hook_dirnames[GD_OP_MAX][256] =
+{
+        [GD_OP_NONE]                    = EMPTY,
+        [GD_OP_CREATE_VOLUME]           = "create",
+        [GD_OP_START_BRICK]             = EMPTY,
+        [GD_OP_STOP_BRICK]              = EMPTY,
+        [GD_OP_DELETE_VOLUME]           = "delete",
+        [GD_OP_START_VOLUME]            = "start",
+        [GD_OP_STOP_VOLUME]             = "stop",
+        [GD_OP_DEFRAG_VOLUME]           = EMPTY,
+        [GD_OP_ADD_BRICK]               = "add-brick",
+        [GD_OP_REMOVE_BRICK]            = "remove-brick",
+        [GD_OP_REPLACE_BRICK]           = EMPTY,
+        [GD_OP_SET_VOLUME]              = "set",
+        [GD_OP_RESET_VOLUME]            = EMPTY,
+        [GD_OP_SYNC_VOLUME]             = EMPTY,
+        [GD_OP_LOG_ROTATE]              = EMPTY,
+        [GD_OP_GSYNC_CREATE]            = "gsync-create",
+        [GD_OP_GSYNC_SET]               = EMPTY,
+        [GD_OP_PROFILE_VOLUME]          = EMPTY,
+        [GD_OP_QUOTA]                   = EMPTY,
+        [GD_OP_STATUS_VOLUME]           = EMPTY,
+        [GD_OP_REBALANCE]               = EMPTY,
+        [GD_OP_HEAL_VOLUME]             = EMPTY,
+        [GD_OP_STATEDUMP_VOLUME]        = EMPTY,
+        [GD_OP_LIST_VOLUME]             = EMPTY,
+        [GD_OP_CLEARLOCKS_VOLUME]       = EMPTY,
+        [GD_OP_DEFRAG_BRICK_VOLUME]     = EMPTY,
+};
+#undef EMPTY
+
+static inline gf_boolean_t
+glusterd_is_hook_enabled (char *script)
+{
+        return (script[0] == 'S');
+}
+
+int
+glusterd_hooks_create_hooks_directory (char *basedir)
+{
+        int  ret                                        = -1;
+        int  op                                         = GD_OP_NONE;
+        int  type                                       = GD_COMMIT_HOOK_NONE;
+        char version_dir[PATH_MAX]                      = {0, };
+        char path[PATH_MAX]                             = {0, };
+        char *cmd_subdir                                = NULL;
+        char type_subdir[GD_COMMIT_HOOK_MAX][256]       = {{0, },
+                                                           "pre",
+                                                           "post"};
+        glusterd_conf_t *priv                           = NULL;
+
+        priv = THIS->private;
+
+        snprintf (path, sizeof (path), "%s/hooks", basedir);
+        ret = mkdir_p (path, 0777, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to create %s due"
+                         "to %s", path, strerror (errno));
+                goto out;
+        }
+
+        GLUSTERD_GET_HOOKS_DIR (version_dir, GLUSTERD_HOOK_VER, priv);
+        ret = mkdir_p (version_dir, 0777, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to create %s due "
+                        "to %s", version_dir, strerror (errno));
+                goto out;
+        }
+
+        for (op = GD_OP_NONE+1; op < GD_OP_MAX; op++) {
+                cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
+                if (strlen (cmd_subdir) == 0)
+                        continue;
+
+                snprintf (path, sizeof (path), "%s/%s", version_dir,
+                          cmd_subdir);
+                ret = mkdir_p (path, 0777, _gf_true);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_CRITICAL,
+                                "Unable to create %s due to %s",
+                                path, strerror (errno));
+                        goto out;
+                }
+
+                for (type = GD_COMMIT_HOOK_PRE; type < GD_COMMIT_HOOK_MAX;
+                     type++) {
+                        snprintf (path, sizeof (path), "%s/%s/%s",
+                                  version_dir, cmd_subdir, type_subdir[type]);
+                        ret = mkdir_p (path, 0777, _gf_true);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_CRITICAL,
+                                        "Unable to create %s due to %s",
+                                        path, strerror (errno));
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+char*
+glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op)
+{
+        GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
+
+        return glusterd_hook_dirnames[op];
+}
+
+int
+glusterd_hooks_set_volume_args (dict_t *dict, runner_t *runner)
+{
+        int     i               = 0;
+        int     count           = 0;
+        int     ret             = -1;
+        char    query[1024]     = {0,};
+        char    *key            = NULL;
+        char    *value          = NULL;
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret)
+                goto out;
+
+        /* This will not happen unless op_ctx
+         * is corrupted*/
+        if (!count)
+                goto out;
+
+        runner_add_arg (runner, "-o");
+        for (i = 1; (ret == 0); i++) {
+                snprintf (query, sizeof (query), "key%d", i);
+                ret = dict_get_str (dict, query, &key);
+                if (ret)
+                       continue;
+
+                snprintf (query, sizeof (query), "value%d", i);
+                ret = dict_get_str (dict, query, &value);
+                if (ret)
+                       continue;
+
+                runner_argprintf (runner, "%s=%s", key, value);
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+glusterd_hooks_add_op_args (runner_t *runner, glusterd_op_t op,
+                            dict_t *op_ctx, glusterd_commit_hook_type_t type)
+{
+        char                   *hooks_args       = NULL;
+        int                     vol_count        = 0;
+        gf_boolean_t            truth            = _gf_false;
+        glusterd_volinfo_t      *voliter         = NULL;
+        glusterd_conf_t         *priv            = NULL;
+        int                     ret              = -1;
+
+        priv = THIS->private;
+        list_for_each_entry (voliter, &priv->volumes,
+                             vol_list) {
+                if (glusterd_is_volume_started (voliter))
+                        vol_count++;
+        }
+
+        ret = 0;
+        switch (op) {
+                case GD_OP_START_VOLUME:
+                        if (type == GD_COMMIT_HOOK_PRE &&
+                            vol_count == 0)
+                                truth = _gf_true;
+
+                        else if (type == GD_COMMIT_HOOK_POST &&
+                                 vol_count == 1)
+                                truth = _gf_true;
+
+                        else
+                                truth = _gf_false;
+
+                        runner_argprintf (runner, "--first=%s",
+                                          truth? "yes":"no");
+                        break;
+
+                case GD_OP_STOP_VOLUME:
+                        if (type == GD_COMMIT_HOOK_PRE &&
+                            vol_count == 1)
+                                truth = _gf_true;
+
+                        else if (type == GD_COMMIT_HOOK_POST &&
+                                 vol_count == 0)
+                                truth = _gf_true;
+
+                        else
+                                truth = _gf_false;
+
+                        runner_argprintf (runner, "--last=%s",
+                                          truth? "yes":"no");
+                        break;
+
+                case GD_OP_SET_VOLUME:
+                        ret = glusterd_hooks_set_volume_args (op_ctx, runner);
+                        break;
+
+                case GD_OP_GSYNC_CREATE:
+                        ret = dict_get_str (op_ctx, "hooks_args", &hooks_args);
+                        if (ret)
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "No Hooks Arguments.");
+                        else
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "Hooks Args = %s", hooks_args);
+                        if (hooks_args)
+                                runner_argprintf (runner, "%s", hooks_args);
+                        break;
+
+                default:
+                        break;
+
+        }
+
+        return ret;
+}
+
+int
+glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                          glusterd_commit_hook_type_t type)
+{
+        xlator_t        *this                   = NULL;
+        glusterd_conf_t *priv                   = NULL;
+        runner_t        runner                  = {0, };
+        struct dirent   *entry                  = NULL;
+        DIR             *hookdir                = NULL;
+        char            *volname                = NULL;
+        char            **lines                 = NULL;
+        int             N                       = 8; /*arbitrary*/
+        int             lineno                  = 0;
+        int             line_count              = 0;
+        int             ret                     = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        ret = dict_get_str (op_ctx, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to get volname "
+                        "from operation context");
+                goto out;
+        }
+
+        hookdir = opendir (hooks_path);
+        if (!hookdir) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s, due "
+                        "to %s", hooks_path, strerror (errno));
+                goto out;
+        }
+
+        lines = GF_CALLOC (1, N * sizeof (*lines), gf_gld_mt_charptr);
+        if (!lines) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = -1;
+        line_count = 0;
+        glusterd_for_each_entry (entry, hookdir);
+        while (entry) {
+                if (line_count == N-1) {
+                        N *= 2;
+                        lines = GF_REALLOC (lines, N * sizeof (char *));
+                        if (!lines)
+                                goto out;
+                }
+
+                if (glusterd_is_hook_enabled (entry->d_name)) {
+                        lines[line_count] = gf_strdup (entry->d_name);
+                        line_count++;
+                }
+
+                glusterd_for_each_entry (entry, hookdir);
+        }
+
+        lines[line_count] = NULL;
+        lines = GF_REALLOC (lines, (line_count + 1) * sizeof (char *));
+        if (!lines)
+                goto out;
+
+        qsort (lines, line_count, sizeof (*lines), glusterd_compare_lines);
+
+        for (lineno = 0; lineno < line_count; lineno++) {
+
+                runinit (&runner);
+                runner_argprintf (&runner, "%s/%s", hooks_path, lines[lineno]);
+                /*Add future command line arguments to hook scripts below*/
+                runner_argprintf (&runner, "--volname=%s", volname);
+                ret = glusterd_hooks_add_op_args (&runner, op, op_ctx, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to add "
+                                "command specific arguments");
+                        goto out;
+                }
+
+                ret = runner_run_reuse (&runner);
+                if (ret) {
+                        runner_log (&runner, this->name, GF_LOG_ERROR,
+                                    "Failed to execute script");
+                } else {
+                        runner_log (&runner, this->name, GF_LOG_INFO,
+                                    "Ran script");
+                }
+                runner_end (&runner);
+        }
+
+        ret = 0;
+out:
+        if (lines) {
+                for (lineno = 0; lineno < line_count+1; lineno++)
+                        GF_FREE (lines[lineno]);
+
+                GF_FREE (lines);
+        }
+
+        if (hookdir)
+                closedir (hookdir);
+
+        return ret;
+}
+
+int
+glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
+                                  dict_t *op_ctx)
+{
+        int                     ret     = -1;
+        glusterd_hooks_stub_t   *stub   = NULL;
+        glusterd_hooks_private_t *hooks_priv = NULL;
+        glusterd_conf_t         *conf   = NULL;
+
+        conf = THIS->private;
+        hooks_priv = conf->hooks_priv;
+
+        ret = glusterd_hooks_stub_init (&stub, scriptdir, op, op_ctx);
+        if (ret)
+                goto out;
+
+        pthread_mutex_lock (&hooks_priv->mutex);
+        {
+                hooks_priv->waitcount++;
+                list_add_tail (&stub->all_hooks, &hooks_priv->list);
+                pthread_cond_signal (&hooks_priv->cond);
+        }
+        pthread_mutex_unlock (&hooks_priv->mutex);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
+                          glusterd_op_t op, dict_t *op_ctx)
+{
+        int                     ret             = -1;
+        glusterd_hooks_stub_t   *hooks_stub     = NULL;
+
+        GF_ASSERT (stub);
+        if (!stub)
+                goto out;
+
+        hooks_stub = GF_CALLOC (1, sizeof (*hooks_stub),
+                                gf_gld_mt_hooks_stub_t);
+        if (!hooks_stub)
+                goto out;
+
+        INIT_LIST_HEAD (&hooks_stub->all_hooks);
+        hooks_stub->op = op;
+        hooks_stub->scriptdir = gf_strdup (scriptdir);
+        if (!hooks_stub->scriptdir)
+                goto out;
+
+        hooks_stub->op_ctx = dict_copy_with_ref (op_ctx, hooks_stub->op_ctx);
+        if (!hooks_stub->op_ctx)
+                goto out;
+
+        *stub = hooks_stub;
+        ret = 0;
+out:
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to initialize "
+                        "post hooks stub");
+                glusterd_hooks_stub_cleanup (hooks_stub);
+        }
+
+        return ret;
+}
+
+void
+glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub)
+{
+        if (!stub) {
+                gf_log_callingfn (THIS->name, GF_LOG_WARNING,
+                                  "hooks_stub is NULL");
+                return;
+        }
+
+        if (stub->op_ctx)
+                dict_unref (stub->op_ctx);
+
+        GF_FREE (stub->scriptdir);
+
+        GF_FREE (stub);
+}
+
+static void*
+hooks_worker (void *args)
+{
+        glusterd_conf_t *conf = NULL;
+        glusterd_hooks_private_t *hooks_priv = NULL;
+        glusterd_hooks_stub_t *stub = NULL;
+
+        THIS = args;
+        conf = THIS->private;
+        hooks_priv = conf->hooks_priv;
+
+        for (;;) {
+                pthread_mutex_lock (&hooks_priv->mutex);
+                {
+                        while (list_empty (&hooks_priv->list)) {
+                                pthread_cond_wait (&hooks_priv->cond,
+                                                   &hooks_priv->mutex);
+                        }
+                        stub = list_entry (hooks_priv->list.next,
+                                           glusterd_hooks_stub_t,
+                                           all_hooks);
+                        list_del_init (&stub->all_hooks);
+                        hooks_priv->waitcount--;
+
+                }
+                pthread_mutex_unlock (&hooks_priv->mutex);
+
+                glusterd_hooks_run_hooks (stub->scriptdir, stub->op,
+                                          stub->op_ctx, GD_COMMIT_HOOK_POST);
+                glusterd_hooks_stub_cleanup (stub);
+        }
+
+        return NULL;
+}
+
+int
+glusterd_hooks_priv_init (glusterd_hooks_private_t **new)
+{
+        int                      ret            = -1;
+        glusterd_hooks_private_t *hooks_priv    = NULL;
+
+        if (!new)
+                goto out;
+
+        hooks_priv = GF_CALLOC (1, sizeof (*hooks_priv),
+                                gf_gld_mt_hooks_priv_t);
+        if (!hooks_priv)
+                goto out;
+
+        pthread_mutex_init (&hooks_priv->mutex, NULL);
+        pthread_cond_init (&hooks_priv->cond, NULL);
+        INIT_LIST_HEAD (&hooks_priv->list);
+        hooks_priv->waitcount = 0;
+
+        *new = hooks_priv;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_hooks_spawn_worker (xlator_t *this)
+{
+        int                       ret          = -1;
+        glusterd_conf_t           *conf        = NULL;
+        glusterd_hooks_private_t  *hooks_priv  = NULL;
+
+
+        ret = glusterd_hooks_priv_init (&hooks_priv);
+        if (ret)
+                goto out;
+
+        conf = this->private;
+        conf->hooks_priv = hooks_priv;
+        ret = pthread_create (&hooks_priv->worker, NULL, hooks_worker,
+                              (void *)this);
+        if (ret)
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn post "
+                        "hooks worker thread");
+out:
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.h b/xlators/mgmt/glusterd/src/glusterd-hooks.h
new file mode 100644
index 000000000..c597ddd2a
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.h
@@ -0,0 +1,89 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_HOOKS_H_
+#define _GLUSTERD_HOOKS_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <fnmatch.h>
+
+#define GLUSTERD_GET_HOOKS_DIR(path, version, priv) \
+        snprintf (path, PATH_MAX, "%s/hooks/%d", priv->workdir,\
+                  version);
+
+#define GLUSTERD_HOOK_VER       1
+
+#define GD_HOOKS_SPECIFIC_KEY   "user.*"
+
+typedef enum glusterd_commit_hook_type {
+        GD_COMMIT_HOOK_NONE = 0,
+        GD_COMMIT_HOOK_PRE,
+        GD_COMMIT_HOOK_POST,
+        GD_COMMIT_HOOK_MAX
+} glusterd_commit_hook_type_t;
+
+typedef struct hooks_private {
+        struct list_head        list;
+        int                     waitcount; //debug purposes
+        pthread_mutex_t         mutex;
+        pthread_cond_t          cond;
+        pthread_t               worker;
+} glusterd_hooks_private_t;
+
+typedef struct hooks_stub {
+        struct list_head        all_hooks;
+        char                    *scriptdir;
+        glusterd_op_t           op;
+        dict_t                  *op_ctx;
+
+} glusterd_hooks_stub_t;
+
+
+static inline gf_boolean_t
+is_key_glusterd_hooks_friendly (char *key)
+{
+        gf_boolean_t is_friendly = _gf_false;
+
+        /* This is very specific to hooks friendly behavior */
+        if (fnmatch (GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "user namespace key %s", key);
+                is_friendly = _gf_true;
+        }
+
+        return is_friendly;
+}
+
+int
+glusterd_hooks_create_hooks_directory (char *basedir);
+
+char *
+glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op);
+
+int
+glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                          glusterd_commit_hook_type_t type);
+int
+glusterd_hooks_spawn_worker (xlator_t *this);
+
+int
+glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
+                          glusterd_op_t op, dict_t *op_ctx);
+void
+glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub);
+
+int
+glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
+                                      dict_t *op_ctx);
+int
+glusterd_hooks_priv_init (glusterd_hooks_private_t **new);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.c b/xlators/mgmt/glusterd/src/glusterd-locks.c
new file mode 100644
index 000000000..0af2a186f
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.c
@@ -0,0 +1,637 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "common-utils.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "glusterd-locks.h"
+#include "run.h"
+#include "syscall.h"
+
+#include <signal.h>
+
+#define GF_MAX_LOCKING_ENTITIES 2
+
+/* Valid entities that the mgmt_v3 lock can hold locks upon    *
+ * To add newer entities to be locked, we can just add more    *
+ * entries to this table along with the type and default value */
+valid_entities   valid_types[] = {
+        { "vol",  _gf_true  },
+        { "snap", _gf_false },
+        { NULL              },
+};
+
+static dict_t *mgmt_v3_lock;
+
+/* Checks if the lock request is for a valid entity */
+gf_boolean_t
+glusterd_mgmt_v3_is_type_valid (char *type)
+{
+        int32_t         i   = 0;
+        gf_boolean_t    ret = _gf_false;
+
+        GF_ASSERT (type);
+
+        for (i = 0; valid_types[i].type; i++) {
+                if (!strcmp (type, valid_types[i].type)) {
+                        ret = _gf_true;
+                        break;
+                }
+        }
+
+        return ret;
+}
+
+/* Initialize the global mgmt_v3 lock list(dict) when
+ * glusterd is spawned */
+int32_t
+glusterd_mgmt_v3_lock_init ()
+{
+        int32_t ret = -1;
+
+        mgmt_v3_lock = dict_new ();
+        if (!mgmt_v3_lock) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+/* Destroy the global mgmt_v3 lock list(dict) when
+ * glusterd cleanup is performed */
+void
+glusterd_mgmt_v3_lock_fini ()
+{
+        if (mgmt_v3_lock)
+                dict_destroy (mgmt_v3_lock);
+}
+
+int32_t
+glusterd_get_mgmt_v3_lock_owner (char *key, uuid_t *uuid)
+{
+        int32_t             ret      = -1;
+        mgmt_v3_lock_obj   *lock_obj = NULL;
+        uuid_t              no_owner = {"\0"};
+        xlator_t           *this = NULL;
+
+        GF_ASSERT(THIS);
+        this = THIS;
+
+        if (!key || !uuid) {
+                gf_log (this->name, GF_LOG_ERROR, "key or uuid is null.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(mgmt_v3_lock, key, (void **) &lock_obj);
+        if (!ret)
+                uuid_copy (*uuid, lock_obj->lock_owner);
+        else
+                uuid_copy (*uuid, no_owner);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* This function is called with the locked_count and type, to   *
+ * release all the acquired locks. */
+static int32_t
+glusterd_release_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
+                                            int32_t locked_count,
+                                            char *type)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        i                     = -1;
+        int32_t        op_ret                = 0;
+        int32_t        ret                   = -1;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        if (locked_count == 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No %s locked as part of this transaction",
+                        type);
+                goto out;
+        }
+
+        /* Release all the locks held */
+        for (i = 0; i < locked_count; i++) {
+                snprintf (name_buf, sizeof(name_buf),
+                          "%sname%d", type, i+1);
+
+                /* Looking for volname1, volname2 or snapname1, *
+                 * as key in the dict snapname2 */
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s locked_count = %d",
+                                name_buf, locked_count);
+                        op_ret = ret;
+                        continue;
+                }
+
+                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release lock for %s.",
+                                name);
+                        op_ret = ret;
+               }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", op_ret);
+        return op_ret;
+}
+
+/* Given the count and type of the entity this function acquires     *
+ * locks on multiple elements of the same entity. For example:       *
+ * If type is "vol" this function tries to acquire locks on multiple *
+ * volumes */
+static int32_t
+glusterd_acquire_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
+                                            int32_t count, char *type)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        i                     = -1;
+        int32_t        ret                   = -1;
+        int32_t        locked_count          = 0;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        /* Locking one element after other */
+        for (i = 0; i < count; i++) {
+                snprintf (name_buf, sizeof(name_buf),
+                          "%sname%d", type, i+1);
+
+                /* Looking for volname1, volname2 or snapname1, *
+                 * as key in the dict snapname2 */
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s count = %d",
+                                name_buf, count);
+                        break;
+                }
+
+                ret = glusterd_mgmt_v3_lock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock for %s %s "
+                                "on behalf of %s. Reversing "
+                                "this transaction", type, name,
+                                uuid_utoa(uuid));
+                        break;
+                }
+                locked_count++;
+        }
+
+        if (count == locked_count) {
+                /* If all locking ops went successfuly, return as success */
+                ret = 0;
+                goto out;
+        }
+
+        /* If we failed to lock one element, unlock others and return failure */
+        ret = glusterd_release_multiple_locks_per_entity (dict, uuid,
+                                                          locked_count,
+                                                          type);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to release multiple %s locks",
+                        type);
+        }
+        ret = -1;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Given the type of entity, this function figures out if it should unlock a   *
+ * single element of multiple elements of the said entity. For example:        *
+ * if the type is "vol", this function will accordingly unlock a single volume *
+ * or multiple volumes */
+static int32_t
+glusterd_mgmt_v3_unlock_entity (dict_t *dict, uuid_t uuid, char *type,
+                                gf_boolean_t default_value)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        count                 = -1;
+        int32_t        ret                   = -1;
+        gf_boolean_t   hold_locks            = _gf_false;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
+        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
+
+        if (hold_locks == _gf_false) {
+                /* Locks were not held for this particular entity *
+                 * Hence nothing to release */
+                ret = 0;
+                goto out;
+        }
+
+        /* Looking for volcount or snapcount in the dict */
+        snprintf (name_buf, sizeof(name_buf), "%scount", type);
+        ret = dict_get_int32 (dict, name_buf, &count);
+        if (ret) {
+                /* count is not present. Only one *
+                 * element name needs to be unlocked */
+                snprintf (name_buf, sizeof(name_buf), "%sname",
+                          type);
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch %sname", type);
+                        goto out;
+                }
+
+                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release lock for %s %s "
+                                "on behalf of %s.", type, name,
+                                uuid_utoa(uuid));
+                        goto out;
+                }
+        } else {
+                /* Unlocking one element name after another */
+                ret = glusterd_release_multiple_locks_per_entity (dict,
+                                                                  uuid,
+                                                                  count,
+                                                                  type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release all %s locks", type);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Given the type of entity, this function figures out if it should lock a   *
+ * single element or multiple elements of the said entity. For example:      *
+ * if the type is "vol", this function will accordingly lock a single volume *
+ * or multiple volumes */
+static int32_t
+glusterd_mgmt_v3_lock_entity (dict_t *dict, uuid_t uuid, char *type,
+                              gf_boolean_t default_value)
+{
+        char           name_buf[PATH_MAX]    = "";
+        char          *name                  = NULL;
+        int32_t        count                 = -1;
+        int32_t        ret                   = -1;
+        gf_boolean_t   hold_locks            = _gf_false;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT (dict);
+        GF_ASSERT (type);
+
+        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
+        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
+
+        if (hold_locks == _gf_false) {
+                /* Not holding locks for this particular entity */
+                ret = 0;
+                goto out;
+        }
+
+        /* Looking for volcount or snapcount in the dict */
+        snprintf (name_buf, sizeof(name_buf), "%scount", type);
+        ret = dict_get_int32 (dict, name_buf, &count);
+        if (ret) {
+                /* count is not present. Only one *
+                 * element name needs to be locked */
+                snprintf (name_buf, sizeof(name_buf), "%sname",
+                          type);
+                ret = dict_get_str (dict, name_buf, &name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch %sname", type);
+                        goto out;
+                }
+
+                ret = glusterd_mgmt_v3_lock (name, uuid, type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock for %s %s "
+                                "on behalf of %s.", type, name,
+                                uuid_utoa(uuid));
+                        goto out;
+                }
+        } else {
+                /* Locking one element name after another */
+                ret = glusterd_acquire_multiple_locks_per_entity (dict,
+                                                                  uuid,
+                                                                  count,
+                                                                  type);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire all %s locks", type);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Try to release locks of multiple entities like *
+ * volume, snaps etc. */
+int32_t
+glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid)
+{
+        int32_t        i                     = -1;
+        int32_t        ret                   = -1;
+        int32_t        op_ret                = 0;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "dict is null.");
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; valid_types[i].type; i++) {
+                ret = glusterd_mgmt_v3_unlock_entity
+                                            (dict, uuid,
+                                             valid_types[i].type,
+                                             valid_types[i].default_value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to unlock all %s",
+                                valid_types[i].type);
+                        op_ret = ret;
+                }
+        }
+
+        ret = op_ret;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Try to acquire locks on multiple entities like *
+ * volume, snaps etc. */
+int32_t
+glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid)
+{
+        int32_t        i                     = -1;
+        int32_t        ret                   = -1;
+        int32_t        locked_count          = 0;
+        xlator_t      *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "dict is null.");
+                ret = -1;
+                goto out;
+        }
+
+        /* Locking one entity after other */
+        for (i = 0; valid_types[i].type; i++) {
+                ret = glusterd_mgmt_v3_lock_entity
+                                            (dict, uuid,
+                                             valid_types[i].type,
+                                             valid_types[i].default_value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to lock all %s",
+                                valid_types[i].type);
+                        break;
+                }
+                locked_count++;
+        }
+
+        if (locked_count == GF_MAX_LOCKING_ENTITIES) {
+                /* If all locking ops went successfuly, return as success */
+                ret = 0;
+                goto out;
+        }
+
+        /* If we failed to lock one entity, unlock others and return failure */
+        for (i = 0; i < locked_count; i++) {
+                ret = glusterd_mgmt_v3_unlock_entity
+                                              (dict, uuid,
+                                               valid_types[i].type,
+                                               valid_types[i].default_value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to unlock all %s",
+                                valid_types[i].type);
+                }
+        }
+        ret = -1;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, char *type)
+{
+        char                 key[PATH_MAX] = "";
+        int32_t              ret           = -1;
+        mgmt_v3_lock_obj    *lock_obj      = NULL;
+        gf_boolean_t         is_valid      = _gf_true;
+        uuid_t               owner         = {0};
+        xlator_t            *this          = NULL;
+
+        GF_ASSERT(THIS);
+        this = THIS;
+
+        if (!name || !type) {
+                gf_log (this->name, GF_LOG_ERROR, "name or type is null.");
+                ret = -1;
+                goto out;
+        }
+
+        is_valid = glusterd_mgmt_v3_is_type_valid (type);
+        if (is_valid != _gf_true) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid entity. Cannot perform locking "
+                        "operation on %s types", type);
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (key, sizeof(key), "%s_%s", name, type);
+        if (ret != strlen(name) + 1 + strlen(type)) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create key");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Trying to acquire lock of %s %s for %s as %s",
+                type, name, uuid_utoa (uuid), key);
+
+        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to get mgmt_v3 lock owner");
+                goto out;
+        }
+
+        /* If the lock has already been held for the given volume
+         * we fail */
+        if (!uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR, "Lock for %s held by %s",
+                        name, uuid_utoa (owner));
+                ret = -1;
+                goto out;
+        }
+
+        lock_obj = GF_CALLOC (1, sizeof(mgmt_v3_lock_obj),
+                              gf_common_mt_mgmt_v3_lock_obj_t);
+        if (!lock_obj) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (lock_obj->lock_owner, uuid);
+
+        ret = dict_set_bin (mgmt_v3_lock, key, lock_obj,
+                            sizeof(*lock_obj));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set lock owner in mgmt_v3 lock");
+                if (lock_obj)
+                        GF_FREE (lock_obj);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Lock for %s %s successfully held by %s",
+                type, name, uuid_utoa (uuid));
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type)
+{
+        char           key[PATH_MAX] = "";
+        int32_t        ret           = -1;
+        gf_boolean_t   is_valid      = _gf_true;
+        uuid_t         owner         = {0};
+        xlator_t            *this          = NULL;
+
+        GF_ASSERT(THIS);
+        this = THIS;
+
+        if (!name || !type) {
+                gf_log (this->name, GF_LOG_ERROR, "name is null.");
+                ret = -1;
+                goto out;
+        }
+
+        is_valid = glusterd_mgmt_v3_is_type_valid (type);
+        if (is_valid != _gf_true) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid entity. Cannot perform unlocking "
+                        "operation on %s types", type);
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (key, sizeof(key), "%s_%s",
+                        name, type);
+        if (ret != strlen(name) + 1 + strlen(type)) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create key");
+                ret = -1;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Trying to release lock of %s %s for %s as %s",
+                type, name, uuid_utoa (uuid), key);
+
+        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to get mgmt_v3 lock owner");
+                goto out;
+        }
+
+        if (uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Lock for %s %s not held", type, name);
+                ret = -1;
+                goto out;
+        }
+
+        ret = uuid_compare (uuid, owner);
+        if (ret) {
+
+                gf_log (this->name, GF_LOG_ERROR, "Lock owner mismatch. "
+                        "Lock for %s %s held by %s",
+                        type, name, uuid_utoa (owner));
+                goto out;
+        }
+
+        /* Removing the mgmt_v3 lock from the global list */
+        dict_del (mgmt_v3_lock, key);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Lock for %s %s successfully released",
+                type, name);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.h b/xlators/mgmt/glusterd/src/glusterd-locks.h
new file mode 100644
index 000000000..83eb8c997
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.h
@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_LOCKS_H_
+#define _GLUSTERD_LOCKS_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+typedef struct mgmt_v3_lock_object_ {
+        uuid_t              lock_owner;
+} mgmt_v3_lock_obj;
+
+typedef struct mgmt_v3_lock_valid_entities {
+        char          *type;          /* Entity type like vol, snap */
+        gf_boolean_t   default_value; /* The default value that  *
+                                       * determines if the locks *
+                                       * should be held for that *
+                                       * entity */
+} valid_entities;
+
+int32_t
+glusterd_mgmt_v3_lock_init ();
+
+void
+glusterd_mgmt_v3_lock_fini ();
+
+int32_t
+glusterd_get_mgmt_v3_lock_owner (char *volname, uuid_t *uuid);
+
+int32_t
+glusterd_mgmt_v3_lock (const char *key, uuid_t uuid, char *type);
+
+int32_t
+glusterd_mgmt_v3_unlock (const char *key, uuid_t uuid, char *type);
+
+int32_t
+glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid);
+
+int32_t
+glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-log-ops.c b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
new file mode 100644
index 000000000..33bd95c03
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
@@ -0,0 +1,271 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "common-utils.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+
+#include <signal.h>
+
+int
+__glusterd_handle_log_rotate (rpcsvc_request_t *req)
+{
+        int32_t                 ret     = -1;
+        gf_cli_req              cli_req = {{0,}};
+        dict_t                 *dict    = NULL;
+        glusterd_op_t           cli_op = GD_OP_LOG_ROTATE;
+        char                   *volname = NULL;
+        char                   msg[2048] = {0,};
+        xlator_t               *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Received log rotate req "
+                "for volume %s", volname);
+
+        ret = dict_set_uint64 (dict, "rotate-key", (uint64_t)time (NULL));
+        if (ret)
+                goto out;
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_LOG_ROTATE, dict);
+
+out:
+        if (ret) {
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, msg);
+        }
+
+        free (cli_req.dict.dict_val);
+        return ret;
+}
+
+int
+glusterd_handle_log_rotate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_log_rotate);
+}
+
+/* op-sm */
+int
+glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        gf_boolean_t                            exists = _gf_false;
+        char                                    msg[2048] = {0};
+        char                                    *brick = NULL;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (!exists) {
+                snprintf (msg, sizeof (msg), "Volume %s does not exist",
+                          volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        if (_gf_false == glusterd_is_volume_started (volinfo)) {
+                snprintf (msg, sizeof (msg), "Volume %s needs to be started before"
+                          " log rotate.", volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "brick", &brick);
+        /* If no brick is specified, do log-rotate for
+           all the bricks in the volume */
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo, NULL);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Incorrect brick %s "
+                          "for volume %s", brick, volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+
+int
+glusterd_op_log_rotate (dict_t *dict)
+{
+        int                   ret                = -1;
+        glusterd_conf_t      *priv               = NULL;
+        glusterd_volinfo_t   *volinfo            = NULL;
+        glusterd_brickinfo_t *brickinfo          = NULL;
+        xlator_t             *this               = NULL;
+        char                 *volname            = NULL;
+        char                 *brick              = NULL;
+        char                  logfile[PATH_MAX]  = {0,};
+        char                  pidfile[PATH_MAX]  = {0,};
+        FILE                 *file               = NULL;
+        pid_t                 pid                = 0;
+        uint64_t              key                = 0;
+        int                   valid_brick        = 0;
+        glusterd_brickinfo_t *tmpbrkinfo         = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "volname not found");
+                goto out;
+        }
+
+        ret = dict_get_uint64 (dict, "rotate-key", &key);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "rotate key not found");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "brick", &brick);
+        /* If no brick is specified, do log-rotate for
+           all the bricks in the volume */
+        if (ret)
+                goto cont;
+
+        ret = glusterd_brickinfo_new_from_brick (brick, &tmpbrkinfo);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "cannot get brickinfo from brick");
+                goto out;
+        }
+
+cont:
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret)
+                goto out;
+
+        ret = -1;
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                if (brick &&
+                    (strcmp (tmpbrkinfo->hostname, brickinfo->hostname) ||
+                     strcmp (tmpbrkinfo->path,brickinfo->path)))
+                        continue;
+
+                valid_brick = 1;
+
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
+                file = fopen (pidfile, "r+");
+                if (!file) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
+                                pidfile);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = fscanf (file, "%d", &pid);
+                if (ret <= 0) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to read pidfile: %s",
+                                pidfile);
+                        ret = -1;
+                        goto out;
+                }
+                fclose (file);
+                file = NULL;
+
+                snprintf (logfile, PATH_MAX, "%s.%"PRIu64,
+                          brickinfo->logfile, key);
+
+                ret = rename (brickinfo->logfile, logfile);
+                if (ret)
+                        gf_log ("", GF_LOG_WARNING, "rename failed");
+
+                ret = kill (pid, SIGHUP);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to SIGHUP to %d", pid);
+                        goto out;
+                }
+                ret = 0;
+
+                /* If request was for brick, only one iteration is enough */
+                if (brick)
+                        break;
+        }
+
+        if (ret && !valid_brick)
+                ret = 0;
+
+out:
+        if (tmpbrkinfo)
+                glusterd_brickinfo_delete (tmpbrkinfo);
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
new file mode 100644
index 000000000..e6f6a0333
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -0,0 +1,75 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __GLUSTERD_MEM_TYPES_H__
+#define __GLUSTERD_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+typedef enum gf_gld_mem_types_ {
+        gf_gld_mt_dir_entry_t                   = gf_common_mt_end + 1,
+        gf_gld_mt_volfile_ctx                   = gf_common_mt_end + 2,
+        gf_gld_mt_glusterd_state_t              = gf_common_mt_end + 3,
+        gf_gld_mt_glusterd_conf_t               = gf_common_mt_end + 4,
+        gf_gld_mt_locker                        = gf_common_mt_end + 5,
+        gf_gld_mt_string                        = gf_common_mt_end + 6,
+        gf_gld_mt_lock_table                    = gf_common_mt_end + 7,
+        gf_gld_mt_char                          = gf_common_mt_end + 8,
+        gf_gld_mt_glusterd_connection_t         = gf_common_mt_end + 9,
+        gf_gld_mt_resolve_comp                  = gf_common_mt_end + 10,
+        gf_gld_mt_peerinfo_t                    = gf_common_mt_end + 11,
+        gf_gld_mt_friend_sm_event_t             = gf_common_mt_end + 12,
+        gf_gld_mt_friend_req_ctx_t              = gf_common_mt_end + 13,
+        gf_gld_mt_friend_update_ctx_t           = gf_common_mt_end + 14,
+        gf_gld_mt_op_sm_event_t                 = gf_common_mt_end + 15,
+        gf_gld_mt_op_lock_ctx_t                 = gf_common_mt_end + 16,
+        gf_gld_mt_op_stage_ctx_t                = gf_common_mt_end + 17,
+        gf_gld_mt_op_commit_ctx_t               = gf_common_mt_end + 18,
+        gf_gld_mt_mop_stage_req_t               = gf_common_mt_end + 19,
+        gf_gld_mt_probe_ctx_t                   = gf_common_mt_end + 20,
+        gf_gld_mt_create_volume_ctx_t           = gf_common_mt_end + 21,
+        gf_gld_mt_start_volume_ctx_t            = gf_common_mt_end + 22,
+        gf_gld_mt_stop_volume_ctx_t             = gf_common_mt_end + 23,
+        gf_gld_mt_delete_volume_ctx_t           = gf_common_mt_end + 24,
+        gf_gld_mt_glusterd_volinfo_t            = gf_common_mt_end + 25,
+        gf_gld_mt_glusterd_brickinfo_t          = gf_common_mt_end + 26,
+        gf_gld_mt_peer_hostname_t               = gf_common_mt_end + 27,
+        gf_gld_mt_ifreq                         = gf_common_mt_end + 28,
+        gf_gld_mt_store_handle_t                = gf_common_mt_end + 29,
+        gf_gld_mt_store_iter_t                  = gf_common_mt_end + 30,
+        gf_gld_mt_defrag_info                   = gf_common_mt_end + 31,
+        gf_gld_mt_log_filename_ctx_t            = gf_common_mt_end + 32,
+        gf_gld_mt_log_locate_ctx_t              = gf_common_mt_end + 33,
+        gf_gld_mt_log_rotate_ctx_t              = gf_common_mt_end + 34,
+        gf_gld_mt_peerctx_t                     = gf_common_mt_end + 35,
+        gf_gld_mt_sm_tr_log_t                   = gf_common_mt_end + 36,
+        gf_gld_mt_pending_node_t                = gf_common_mt_end + 37,
+        gf_gld_mt_brick_rsp_ctx_t               = gf_common_mt_end + 38,
+        gf_gld_mt_mop_brick_req_t               = gf_common_mt_end + 39,
+        gf_gld_mt_op_allack_ctx_t               = gf_common_mt_end + 40,
+        gf_gld_mt_linearr                       = gf_common_mt_end + 41,
+        gf_gld_mt_linebuf                       = gf_common_mt_end + 42,
+        gf_gld_mt_mount_pattern                 = gf_common_mt_end + 43,
+        gf_gld_mt_mount_comp_container          = gf_common_mt_end + 44,
+        gf_gld_mt_mount_component               = gf_common_mt_end + 45,
+        gf_gld_mt_mount_spec                    = gf_common_mt_end + 46,
+        gf_gld_mt_georep_meet_spec              = gf_common_mt_end + 47,
+        gf_gld_mt_nodesrv_t                     = gf_common_mt_end + 48,
+        gf_gld_mt_charptr                       = gf_common_mt_end + 49,
+        gf_gld_mt_hooks_stub_t                  = gf_common_mt_end + 50,
+        gf_gld_mt_hooks_priv_t                  = gf_common_mt_end + 51,
+        gf_gld_mt_mop_commit_req_t              = gf_common_mt_end + 52,
+        gf_gld_mt_int                           = gf_common_mt_end + 53,
+        gf_gld_mt_snap_t                        = gf_common_mt_end + 54,
+        gf_gld_mt_missed_snapinfo_t             = gf_common_mt_end + 55,
+        gf_gld_mt_end                           = gf_common_mt_end + 56,
+} gf_gld_mem_types_t;
+#endif
+
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
new file mode 100644
index 000000000..a5f38ce9c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
@@ -0,0 +1,924 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+/* rpc related syncops */
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+#include "glusterd-syncop.h"
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-op-sm.h"
+
+static int
+glusterd_mgmt_v3_null (rpcsvc_request_t *req)
+{
+        return 0;
+}
+
+static int
+glusterd_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, int32_t status)
+{
+
+        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
+        int                           ret   = -1;
+        xlator_t                     *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+
+        glusterd_get_uuid (&rsp.uuid);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to mgmt_v3 lock, ret: %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_synctasked_mgmt_v3_lock (rpcsvc_request_t *req,
+                                  gd1_mgmt_v3_lock_req *lock_req,
+                                  glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                         ret         = -1;
+        xlator_t                       *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (ctx);
+        GF_ASSERT (ctx->dict);
+
+        /* Trying to acquire multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_lock (ctx->dict, ctx->uuid);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to acquire mgmt_v3 locks for %s",
+                         uuid_utoa (ctx->uuid));
+
+        ret = glusterd_mgmt_v3_lock_send_resp (req, ret);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_op_state_machine_mgmt_v3_lock (rpcsvc_request_t *req,
+                                        gd1_mgmt_v3_lock_req *lock_req,
+                                        glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                         ret         = -1;
+        xlator_t                       *this        = NULL;
+        glusterd_op_info_t              txn_op_info = {{0},};
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        glusterd_txn_opinfo_init (&txn_op_info, NULL, &lock_req->op,
+                                  ctx->dict, req);
+
+        ret = glusterd_set_txn_opinfo (&lock_req->txn_id, &txn_op_info);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+                goto out;
+        }
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK,
+                                           &lock_req->txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_LOCK");
+
+out:
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_mgmt_v3_lock_fn (rpcsvc_request_t *req)
+{
+        gd1_mgmt_v3_lock_req            lock_req    = {{0},};
+        int32_t                         ret         = -1;
+        glusterd_peerinfo_t            *peerinfo    = NULL;
+        glusterd_op_lock_ctx_t         *ctx         = NULL;
+        xlator_t                       *this        = NULL;
+        gf_boolean_t                    is_synctasked = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received mgmt_v3 lock req "
+                "from uuid: %s", uuid_utoa (lock_req.uuid));
+
+        if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (lock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+        if (!ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, lock_req.uuid);
+        ctx->req = req;
+
+        ctx->dict = dict_new ();
+        if (!ctx->dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (lock_req.dict.dict_val,
+                                lock_req.dict.dict_len, &ctx->dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        is_synctasked = dict_get_str_boolean (ctx->dict,
+                                              "is_synctasked", _gf_false);
+        if (is_synctasked)
+                ret = glusterd_synctasked_mgmt_v3_lock (req, &lock_req, ctx);
+        else
+                ret = glusterd_op_state_machine_mgmt_v3_lock (req, &lock_req,
+                                                              ctx);
+
+out:
+
+        if (ret) {
+                if (ctx->dict)
+                        dict_unref (ctx->dict);
+                if (ctx)
+                        GF_FREE (ctx);
+        }
+
+        free (lock_req.dict.dict_val);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_pre_validate_send_resp (rpcsvc_request_t   *req,
+                                        int32_t op, int32_t status,
+                                        char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_pre_val_rsp          rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to pre validation, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_pre_validate_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_pre_val_req          op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode pre validation "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_pre_validate_fn (op_req.op, dict, &op_errstr,
+                                         rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pre Validation failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_pre_validate_send_resp (req, op_req.op,
+                                                      ret, op_errstr,
+                                                      rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send Pre Validation "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_brick_op_send_resp (rpcsvc_request_t   *req,
+                                    int32_t op, int32_t status,
+                                    char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_brick_op_rsp         rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to brick op, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_brick_op_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_brick_op_req         op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode brick op "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_brick_op_fn (op_req.op, dict, &op_errstr,
+                                     rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick Op failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_brick_op_send_resp (req, op_req.op,
+                                                  ret, op_errstr,
+                                                  rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send brick op "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_commit_send_resp (rpcsvc_request_t   *req,
+                                  int32_t op, int32_t status,
+                                  char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_commit_rsp           rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Responded to commit, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_commit_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_commit_req           op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_commit_fn (op_req.op, dict, &op_errstr,
+                                   rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "commit failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_commit_send_resp (req, op_req.op,
+                                                ret, op_errstr,
+                                                rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send commit "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_post_validate_send_resp (rpcsvc_request_t   *req,
+                                         int32_t op, int32_t status,
+                                         char *op_errstr, dict_t *rsp_dict)
+{
+        gd1_mgmt_v3_post_val_rsp         rsp      = {{0},};
+        int                             ret      = -1;
+        xlator_t                       *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        glusterd_get_uuid (&rsp.uuid);
+        rsp.op = op;
+        if (op_errstr)
+                rsp.op_errstr = op_errstr;
+        else
+                rsp.op_errstr = "";
+
+        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
+                                           &rsp.dict.dict_len);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get serialized length of dict");
+                goto out;
+        }
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+
+        GF_FREE (rsp.dict.dict_val);
+out:
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to post validation, ret: %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_post_validate_fn (rpcsvc_request_t *req)
+{
+        int32_t                         ret       = -1;
+        gd1_mgmt_v3_post_val_req         op_req    = {{0},};
+        glusterd_peerinfo_t            *peerinfo  = NULL;
+        xlator_t                       *this      = NULL;
+        char                           *op_errstr = NULL;
+        dict_t                         *dict      = NULL;
+        dict_t                         *rsp_dict  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &op_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode post validation "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (glusterd_friend_find_by_uuid (op_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (op_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_unserialize (op_req.dict.dict_val,
+                                op_req.dict.dict_len, &dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        ret = gd_mgmt_v3_post_validate_fn (op_req.op, op_req.op_ret, dict,
+                                           &op_errstr, rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Post Validation failed on operation %s",
+                        gd_op_list[op_req.op]);
+        }
+
+        ret = glusterd_mgmt_v3_post_validate_send_resp (req, op_req.op,
+                                                       ret, op_errstr,
+                                                       rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to send Post Validation "
+                        "response for operation %s",
+                        gd_op_list[op_req.op]);
+                goto out;
+        }
+
+out:
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        free (op_req.dict.dict_val);
+
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
+{
+
+        gd1_mgmt_v3_unlock_rsp          rsp   = {{0},};
+        int                             ret   = -1;
+        xlator_t                       *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        rsp.op_ret = status;
+        if (rsp.op_ret)
+               rsp.op_errno = errno;
+
+        glusterd_get_uuid (&rsp.uuid);
+
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Responded to mgmt_v3 unlock, ret: %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_synctasked_mgmt_v3_unlock (rpcsvc_request_t *req,
+                                    gd1_mgmt_v3_unlock_req *unlock_req,
+                                    glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                         ret         = -1;
+        xlator_t                       *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (ctx);
+
+        /* Trying to release multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_unlock (ctx->dict, ctx->uuid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to release mgmt_v3 locks for %s",
+                        uuid_utoa(ctx->uuid));
+        }
+
+        ret = glusterd_mgmt_v3_unlock_send_resp (req, ret);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+
+static int
+glusterd_op_state_machine_mgmt_v3_unlock (rpcsvc_request_t *req,
+                                          gd1_mgmt_v3_unlock_req *lock_req,
+                                          glusterd_op_lock_ctx_t *ctx)
+{
+        int32_t                           ret      = -1;
+        xlator_t                         *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK,
+                                           &lock_req->txn_id, ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to inject event GD_OP_EVENT_UNLOCK");
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_handle_mgmt_v3_unlock_fn (rpcsvc_request_t *req)
+{
+        gd1_mgmt_v3_unlock_req            lock_req = {{0},};
+        int32_t                           ret      = -1;
+        glusterd_op_lock_ctx_t           *ctx      = NULL;
+        glusterd_peerinfo_t              *peerinfo = NULL;
+        xlator_t                         *this     = NULL;
+        gf_boolean_t                      is_synctasked = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &lock_req,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "request received from peer");
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received mgmt_v3 unlock req "
+                "from uuid: %s", uuid_utoa (lock_req.uuid));
+
+        if (glusterd_friend_find_by_uuid (lock_req.uuid, &peerinfo)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
+                        "belong to the cluster. Ignoring request.",
+                        uuid_utoa (lock_req.uuid));
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+        if (!ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ctx->uuid, lock_req.uuid);
+        ctx->req = req;
+
+        ctx->dict = dict_new ();
+        if (!ctx->dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_unserialize (lock_req.dict.dict_val,
+                                lock_req.dict.dict_len, &ctx->dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to unserialize the dictionary");
+                goto out;
+        }
+
+        is_synctasked = dict_get_str_boolean (ctx->dict,
+                                              "is_synctasked", _gf_false);
+        if (is_synctasked)
+                ret = glusterd_synctasked_mgmt_v3_unlock (req, &lock_req, ctx);
+        else
+                ret = glusterd_op_state_machine_mgmt_v3_unlock (req, &lock_req,
+                                                                ctx);
+
+out:
+
+        if (ret) {
+                if (ctx->dict)
+                        dict_unref (ctx->dict);
+                if (ctx)
+                        GF_FREE (ctx);
+        }
+
+        free (lock_req.dict.dict_val);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_handle_mgmt_v3_lock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_mgmt_v3_lock_fn);
+}
+
+static int
+glusterd_handle_pre_validate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_pre_validate_fn);
+}
+
+static int
+glusterd_handle_brick_op (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_brick_op_fn);
+}
+
+static int
+glusterd_handle_commit (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_commit_fn);
+}
+
+static int
+glusterd_handle_post_validate (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_post_validate_fn);
+}
+
+int
+glusterd_handle_mgmt_v3_unlock (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            glusterd_handle_mgmt_v3_unlock_fn);
+}
+
+rpcsvc_actor_t gd_svc_mgmt_v3_actors[] = {
+        [GLUSTERD_MGMT_V3_NULL]          = { "NULL",           GLUSTERD_MGMT_V3_NULL,          glusterd_mgmt_v3_null,          NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_LOCK]          = { "MGMT_V3_LOCK",   GLUSTERD_MGMT_V3_LOCK,          glusterd_handle_mgmt_v3_lock,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_PRE_VALIDATE]  = { "PRE_VAL",        GLUSTERD_MGMT_V3_PRE_VALIDATE,  glusterd_handle_pre_validate,   NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_BRICK_OP]      = { "BRCK_OP",        GLUSTERD_MGMT_V3_BRICK_OP,      glusterd_handle_brick_op,       NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_COMMIT]        = { "COMMIT",         GLUSTERD_MGMT_V3_COMMIT,        glusterd_handle_commit,         NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_POST_VALIDATE] = { "POST_VAL",       GLUSTERD_MGMT_V3_POST_VALIDATE, glusterd_handle_post_validate,  NULL, 0, DRC_NA},
+        [GLUSTERD_MGMT_V3_UNLOCK]        = { "MGMT_V3_UNLOCK", GLUSTERD_MGMT_V3_UNLOCK,        glusterd_handle_mgmt_v3_unlock, NULL, 0, DRC_NA},
+};
+
+struct rpcsvc_program gd_svc_mgmt_v3_prog = {
+        .progname  = "GlusterD svc mgmt v3",
+        .prognum   = GD_MGMT_V3_PROGRAM,
+        .progver   = GD_MGMT_V3_VERSION,
+        .numactors = GLUSTERD_MGMT_V3_MAXVALUE,
+        .actors    = gd_svc_mgmt_v3_actors,
+        .synctask  = _gf_true,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
new file mode 100644
index 000000000..d52532e54
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
@@ -0,0 +1,1893 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+/* rpc related syncops */
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+#include "glusterd-syncop.h"
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-op-sm.h"
+
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+
+
+static void
+gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                           char *op_errstr, int op_code,
+                           glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char      *peer_str          = NULL;
+        char       err_str[PATH_MAX] = "Please check log file for details.";
+        char       op_err[PATH_MAX]  = "";
+        int32_t    len               = -1;
+        xlator_t  *this              = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (args);
+        GF_ASSERT (uuid);
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, "")) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "Error: %s", op_errstr);
+                        err_str[len] = '\0';
+                }
+
+                switch (op_code){
+                case GLUSTERD_MGMT_V3_LOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Locking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_PRE_VALIDATE:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Pre Validation failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_BRICK_OP:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Brick ops failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_COMMIT:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Commit failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_POST_VALIDATE:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Post Validation failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                case GLUSTERD_MGMT_V3_UNLOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Unlocking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                }
+                op_err[len] = '\0';
+
+                if (args->errstr) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s\n%s", args->errstr,
+                                        op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s", op_err);
+                err_str[len] = '\0';
+
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+int32_t
+gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
+                            char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+               case GD_OP_SNAP:
+               {
+                       ret = glusterd_snapshot_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                       if (ret) {
+                               gf_log (this->name, GF_LOG_WARNING,
+                                       "Snapshot Prevalidate Failed");
+                               goto out;
+                       }
+
+                       break;
+               }
+               default:
+                       break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
+                        char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+        case GD_OP_SNAP:
+        {
+                ret = glusterd_snapshot_brickop (dict, op_errstr, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "snapshot brickop "
+                                "failed");
+                        goto out;
+                }
+                break;
+        }
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
+                      char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+               case GD_OP_SNAP:
+               {
+                       ret = glusterd_snapshot (dict, op_errstr, rsp_dict);
+                       if (ret) {
+                               gf_log (this->name, GF_LOG_WARNING,
+                                       "Snapshot Commit Failed");
+                               goto out;
+                       }
+                       break;
+               }
+               default:
+                       break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                             char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t       ret = -1;
+        xlator_t     *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        switch (op) {
+               case GD_OP_SNAP:
+               {
+                       ret = glusterd_snapshot_postvalidate (dict, op_ret,
+                                                             op_errstr,
+                                                             rsp_dict);
+                       if (ret) {
+                               gf_log (this->name, GF_LOG_WARNING,
+                                       "postvalidate operation failed");
+                               goto out;
+                       }
+                       break;
+               }
+               default:
+                       break;
+        }
+
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_lock_rsp        rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        /* Even though the lock command has failed, while collating the errors
+           (gd_mgmt_v3_collate_errors), args->op_ret and args->op_errno will be
+           used. @args is obtained from frame->local. So before checking the
+           status of the request and going out if its a failure, args should be
+           set to frame->local. Otherwise, while collating args will be NULL.
+           This applies to other phases such as prevalidate, brickop, commit and
+           postvalidate also.
+        */
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_LOCK,
+                                   peerinfo, rsp.uuid);
+        free (rsp.dict.dict_val);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                     int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_lock_cbk_fn);
+}
+
+int
+gd_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
+                 glusterd_peerinfo_t *peerinfo,
+                 struct syncargs *args, uuid_t my_uuid,
+                 uuid_t recv_uuid)
+{
+        gd1_mgmt_v3_lock_req     req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+        int32_t                  ret  = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_LOCK,
+                                        gd_mgmt_v3_lock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_lock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_initiate_lockdown (glusterd_conf_t  *conf, glusterd_op_t op,
+                                    dict_t *dict, char **op_errstr, int npeers,
+                                    gf_boolean_t  *is_acquired)
+{
+        char                *volname    = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (is_acquired);
+
+        peers = &conf->xaction_peers;
+
+        /* Trying to acquire multiple mgmt_v3 locks on local node */
+        ret = glusterd_multiple_mgmt_v3_lock (dict, MY_UUID);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to acquire mgmt_v3 locks on localhost");
+                goto out;
+        }
+
+        *is_acquired = _gf_true;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending mgmt_v3 lock req to other nodes in the cluster */
+        gd_syncargs_init (&args, NULL);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_lock (op, dict, peerinfo, &args,
+                                 MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.errstr)
+                *op_errstr = gf_strdup (args.errstr);
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent lock op req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        if (ret) {
+                if (*op_errstr)
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                *op_errstr);
+
+                if (volname)
+                        ret = gf_asprintf (op_errstr,
+                                           "Another transaction is in progress "
+                                           "for %s. Please try again after "
+                                           "sometime.", volname);
+                else
+                        ret = gf_asprintf (op_errstr,
+                                           "Another transaction is in progress "
+                                           "Please try again after sometime.");
+
+                if (ret == -1)
+                        *op_errstr = NULL;
+
+                ret = -1;
+        }
+
+        return ret;
+}
+
+int
+glusterd_pre_validate_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
+{
+        int32_t              ret      = 0;
+        xlator_t            *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (aggr);
+        GF_ASSERT (rsp);
+
+        switch (op) {
+        case GD_OP_SNAP:
+                ret = glusterd_snap_pre_validate_use_rsp_dict (aggr, rsp);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to aggregate prevalidate "
+                                "response dictionaries.");
+                        goto out;
+                }
+                break;
+        default:
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Invalid op (%s)", gd_op_list[op]);
+
+                break;
+        }
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_pre_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_pre_val_rsp      rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        dict_t                     *rsp_dict   = NULL;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        pthread_mutex_lock (&args->lock_dict);
+        {
+                ret = glusterd_pre_validate_aggr_rsp_dict (rsp.op, args->dict,
+                                                           rsp_dict);
+        }
+        pthread_mutex_unlock (&args->lock_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                if (!rsp.op_ret)
+                        op_ret = ret;
+                else {
+                        op_ret = rsp.op_ret;
+                        op_errno = rsp.op_errno;
+                }
+        } else {
+                op_ret = rsp.op_ret;
+                op_errno = rsp.op_errno;
+        }
+
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                  GLUSTERD_MGMT_V3_PRE_VALIDATE,
+                                  peerinfo, rsp.uuid);
+
+        if (rsp.op_errstr)
+                free (rsp.op_errstr);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_pre_validate_cbk (struct rpc_req *req, struct iovec *iov,
+                             int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_pre_validate_cbk_fn);
+}
+
+int
+gd_mgmt_v3_pre_validate_req (glusterd_op_t op, dict_t *op_ctx,
+                             glusterd_peerinfo_t *peerinfo,
+                             struct syncargs *args, uuid_t my_uuid,
+                             uuid_t recv_uuid)
+{
+        int32_t                  ret   = -1;
+        gd1_mgmt_v3_pre_val_req  req   = {{0},};
+        glusterd_conf_t         *conf  = THIS->private;
+        xlator_t                *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_PRE_VALIDATE,
+                                        gd_mgmt_v3_pre_validate_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_pre_val_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_pre_validate (glusterd_conf_t  *conf, glusterd_op_t op,
+                               dict_t *req_dict, char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Pre Validation on local node */
+        ret = gd_mgmt_v3_pre_validate_fn (op, req_dict, op_errstr,
+                                          rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pre Validation failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Pre-validation failed "
+                                           "on localhost. Please "
+                                           "check log file for details");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        ret = glusterd_pre_validate_aggr_rsp_dict (op, req_dict,
+                                                   rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending Pre Validation req to other nodes in the cluster */
+        gd_syncargs_init (&args, req_dict);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_pre_validate_req (op, req_dict, peerinfo, &args,
+                                             MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Pre Validation failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent pre valaidation req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict,
+                                glusterd_op_t op)
+{
+        int32_t                 ret      = -1;
+        dict_t                 *req_dict = NULL;
+        xlator_t               *this     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (dict);
+
+        req_dict = dict_new ();
+        if (!req_dict)
+                goto out;
+
+        switch (op) {
+                case GD_OP_SNAP:
+                        dict_copy (dict, req_dict);
+                        break;
+                default:
+                        break;
+        }
+
+        *req = req_dict;
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_brick_op_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_brick_op_rsp     rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        /* If the operation failed, then iov can be NULL. So better check the
+           status of the operation and then worry about iov (if the status of
+           the command is success)
+        */
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_BRICK_OP,
+                                   peerinfo, rsp.uuid);
+
+        if (rsp.op_errstr)
+                free (rsp.op_errstr);
+
+        free (rsp.dict.dict_val);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_brick_op_cbk_fn);
+}
+
+int
+gd_mgmt_v3_brick_op_req (glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_peerinfo_t *peerinfo,
+                         struct syncargs *args, uuid_t my_uuid,
+                         uuid_t recv_uuid)
+{
+        int32_t                   ret  = -1;
+        gd1_mgmt_v3_brick_op_req  req  = {{0},};
+        glusterd_conf_t          *conf = THIS->private;
+        xlator_t                 *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_BRICK_OP,
+                                        gd_mgmt_v3_brick_op_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_brick_op_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_brick_op (glusterd_conf_t  *conf, glusterd_op_t op,
+                           dict_t *req_dict, char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Perform brick op on local node */
+        ret = gd_mgmt_v3_brick_op_fn (op, req_dict, op_errstr,
+                                     rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick ops failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Brick ops failed "
+                                           "on localhost. Please "
+                                           "check log file for details");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending brick op req to other nodes in the cluster */
+        gd_syncargs_init (&args, NULL);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_brick_op_req (op, req_dict, peerinfo, &args,
+                                         MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Brick ops failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_commit_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_commit_rsp       rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        dict_t                     *rsp_dict   = NULL;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        pthread_mutex_lock (&args->lock_dict);
+        {
+                ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
+                                                     rsp_dict);
+        }
+        pthread_mutex_unlock (&args->lock_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                if (!rsp.op_ret)
+                        op_ret = ret;
+                else {
+                        op_ret = rsp.op_ret;
+                        op_errno = rsp.op_errno;
+                }
+        } else {
+                op_ret = rsp.op_ret;
+                op_errno = rsp.op_errno;
+        }
+
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                  GLUSTERD_MGMT_V3_COMMIT,
+                                  peerinfo, rsp.uuid);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_commit_cbk (struct rpc_req *req, struct iovec *iov,
+                       int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_commit_cbk_fn);
+}
+
+int
+gd_mgmt_v3_commit_req (glusterd_op_t op, dict_t *op_ctx,
+                       glusterd_peerinfo_t *peerinfo,
+                       struct syncargs *args, uuid_t my_uuid,
+                       uuid_t recv_uuid)
+{
+        int32_t                  ret  = -1;
+        gd1_mgmt_v3_commit_req   req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_COMMIT,
+                                        gd_mgmt_v3_commit_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_commit_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_commit (glusterd_conf_t  *conf, glusterd_op_t op,
+                         dict_t *op_ctx, dict_t *req_dict,
+                         char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Commit on local node */
+        ret = gd_mgmt_v3_commit_fn (op, req_dict, op_errstr,
+                                   rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Commit failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Commit failed "
+                                           "on localhost. Please "
+                                           "check log file for details.");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx,
+                                             rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        "Failed to aggregate response from "
+                        " node/brick");
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending commit req to other nodes in the cluster */
+        gd_syncargs_init (&args, op_ctx);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_commit_req (op, req_dict, peerinfo, &args,
+                                       MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Commit failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent commit req for %s to %d "
+                "peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_post_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_post_val_rsp     rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                  GLUSTERD_MGMT_V3_POST_VALIDATE,
+                                  peerinfo, rsp.uuid);
+        if (rsp.op_errstr)
+                free (rsp.op_errstr);
+
+        free (rsp.dict.dict_val);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_post_validate_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_post_validate_cbk_fn);
+}
+
+int
+gd_mgmt_v3_post_validate_req (glusterd_op_t op, int32_t op_ret, dict_t *op_ctx,
+                              glusterd_peerinfo_t *peerinfo,
+                              struct syncargs *args, uuid_t my_uuid,
+                              uuid_t recv_uuid)
+{
+        int32_t                   ret  = -1;
+        gd1_mgmt_v3_post_val_req  req  = {{0},};
+        glusterd_conf_t          *conf = THIS->private;
+        xlator_t                 *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        req.op_ret = op_ret;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_POST_VALIDATE,
+                                        gd_mgmt_v3_post_validate_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_post_val_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_post_validate (glusterd_conf_t  *conf, glusterd_op_t op,
+                                int32_t op_ret, dict_t *dict, dict_t *req_dict,
+                                char **op_errstr, int npeers)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        dict_t              *rsp_dict   = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (req_dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+        GF_ASSERT (peers);
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create response dictionary");
+                goto out;
+        }
+
+        /* Copy the contents of dict like missed snaps info to req_dict */
+        dict_copy (dict, req_dict);
+
+        /* Post Validation on local node */
+        ret = gd_mgmt_v3_post_validate_fn (op, op_ret, req_dict, op_errstr,
+                                           rsp_dict);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Post Validation failed for "
+                        "operation %s on local node",
+                        gd_op_list[op]);
+
+                if (*op_errstr == NULL) {
+                        ret = gf_asprintf (op_errstr,
+                                           "Post-validation failed "
+                                           "on localhost. Please check "
+                                           "log file for details");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        ret = -1;
+                }
+                goto out;
+        }
+
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending Post Validation req to other nodes in the cluster */
+        gd_syncargs_init (&args, req_dict);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_post_validate_req (op, op_ret, req_dict, peerinfo,
+                                              &args, MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Post Validation failed on peers");
+
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent post valaidation req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int32_t
+gd_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int32_t                     ret        = -1;
+        struct syncargs            *args       = NULL;
+        glusterd_peerinfo_t        *peerinfo   = NULL;
+        gd1_mgmt_v3_unlock_rsp      rsp        = {{0},};
+        call_frame_t               *frame      = NULL;
+        int32_t                     op_ret     = -1;
+        int32_t                     op_errno   = -1;
+        xlator_t                   *this       = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        if (!iov) {
+                gf_log (this->name, GF_LOG_ERROR, "iov is NULL");
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_UNLOCK,
+                                   peerinfo, rsp.uuid);
+        free (rsp.dict.dict_val);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                       int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_mgmt_v3_unlock_cbk_fn);
+}
+
+int
+gd_mgmt_v3_unlock (glusterd_op_t op, dict_t *op_ctx,
+                   glusterd_peerinfo_t *peerinfo,
+                   struct syncargs *args, uuid_t my_uuid,
+                   uuid_t recv_uuid)
+{
+        int32_t                  ret  = -1;
+        gd1_mgmt_v3_unlock_req   req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_UNLOCK,
+                                        gd_mgmt_v3_unlock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_unlock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        GF_FREE (req.dict.dict_val);
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_mgmt_v3_release_peer_locks (glusterd_conf_t  *conf, glusterd_op_t op,
+                                     dict_t *dict, int32_t op_ret,
+                                     char **op_errstr, int npeers,
+                                     gf_boolean_t  is_acquired)
+{
+        int32_t              ret        = -1;
+        int32_t              peer_cnt   = 0;
+        uuid_t               peer_uuid  = {0};
+        xlator_t            *this       = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs      args       = {0};
+        struct list_head    *peers      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (conf);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        peers = &conf->xaction_peers;
+
+        /* If the lock has not been held during this
+         * transaction, do not send unlock requests */
+        if (!is_acquired)
+                goto out;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Sending mgmt_v3 unlock req to other nodes in the cluster */
+        gd_syncargs_init (&args, NULL);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                gd_mgmt_v3_unlock (op, dict, peerinfo, &args,
+                                   MY_UUID, peer_uuid);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unlock failed on peers");
+
+                if (!op_ret && args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent unlock op req for %s "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                      dict_t *dict)
+{
+        int32_t                     ret              = -1;
+        int32_t                     op_ret           = -1;
+        int32_t                     npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        dict_t                      *tmp_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        char                        *op_errstr       = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *originator_uuid = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+        /* Marking the operation as complete synctasked */
+        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set synctasked flag.");
+                goto out;
+        }
+
+        /* Use a copy at local unlock as cli response will be sent before
+         * the unlock and the volname in the dict might be removed */
+        tmp_dict = dict_new();
+        if (!tmp_dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
+                goto out;
+        }
+        dict_copy (dict, tmp_dict);
+
+        /* BUILD PEERS LIST */
+        INIT_LIST_HEAD (&conf->xaction_peers);
+        npeers = gd_build_peers_list  (&conf->peers, &conf->xaction_peers, op);
+
+        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+        ret = glusterd_mgmt_v3_initiate_lockdown (conf, op, dict, &op_errstr,
+                                                  npeers, &is_acquired);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed.");
+                goto out;
+        }
+
+        /* BUILD PAYLOAD */
+        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                goto out;
+        }
+
+        /* PRE-COMMIT VALIDATE PHASE */
+        ret = glusterd_mgmt_v3_pre_validate (conf, op, req_dict,
+                                             &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed");
+                goto out;
+        }
+
+        /* COMMIT OP PHASE */
+        ret = glusterd_mgmt_v3_commit (conf, op, dict, req_dict,
+                                       &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed");
+                goto out;
+        }
+
+        /* POST-COMMIT VALIDATE PHASE */
+        /* As of now, post_validate is not handling any other
+           commands other than snapshot. So as of now, I am
+           sending 0 (op_ret as 0).
+        */
+        ret = glusterd_mgmt_v3_post_validate (conf, op, 0, dict, req_dict,
+                                              &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        op_ret = ret;
+        /* UNLOCK PHASE FOR PEERS*/
+        (void) glusterd_mgmt_v3_release_peer_locks (conf, op, dict,
+                                                    op_ret, &op_errstr,
+                                                    npeers, is_acquired);
+
+        /* LOCAL VOLUME(S) UNLOCK */
+        if (is_acquired) {
+                /* Trying to release multiple mgmt_v3 locks */
+                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release mgmt_v3 locks on localhost");
+                        op_ret = ret;
+                }
+        }
+
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr);
+
+        if (req_dict)
+                dict_unref (req_dict);
+
+        if (tmp_dict)
+                dict_unref (tmp_dict);
+
+        if (op_errstr) {
+                GF_FREE (op_errstr);
+                op_errstr = NULL;
+        }
+
+        return 0;
+}
+
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                       dict_t *dict)
+{
+        int32_t                     ret              = -1;
+        int32_t                     op_ret           = -1;
+        int32_t                     npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        dict_t                      *tmp_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        char                        *op_errstr       = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *originator_uuid = NULL;
+        gf_boolean_t                success          = _gf_false;
+        char                        *tmp_errstr      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        /* Save the MY_UUID as the originator_uuid. This originator_uuid
+         * will be used by is_origin_glusterd() to determine if a node
+         * is the originator node for a command. */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (dict, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+        /* Marking the operation as complete synctasked */
+        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set synctasked flag.");
+                goto out;
+        }
+
+        /* Use a copy at local unlock as cli response will be sent before
+         * the unlock and the volname in the dict might be removed */
+        tmp_dict = dict_new();
+        if (!tmp_dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
+                goto out;
+        }
+        dict_copy (dict, tmp_dict);
+
+        /* BUILD PEERS LIST */
+        INIT_LIST_HEAD (&conf->xaction_peers);
+        npeers = gd_build_peers_list  (&conf->peers, &conf->xaction_peers, op);
+
+        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+        ret = glusterd_mgmt_v3_initiate_lockdown (conf, op, dict, &op_errstr,
+                                                  npeers, &is_acquired);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed.");
+                goto out;
+        }
+
+        /* BUILD PAYLOAD */
+        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                goto out;
+        }
+
+        /* PRE-COMMIT VALIDATE PHASE */
+        ret = glusterd_mgmt_v3_pre_validate (conf, op, req_dict,
+                                            &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed");
+                goto out;
+        }
+
+        /* BRICK OP PHASE for initiating barrier*/
+        ret = dict_set_int32 (req_dict, "barrier", 1);
+        if (ret)
+                goto out;
+        ret = glusterd_mgmt_v3_brick_op (conf, op, req_dict,
+                                        &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed");
+                goto unbarrier;
+        }
+
+        /* COMMIT OP PHASE */
+        /* TODO: As of now, the plan is to do quorum check before sending the
+           commit fop and if the quorum succeeds, then commit is sent to all
+           the other glusterds.
+           snap create functionality now creates the in memory and on disk
+           objects for the snapshot (marking them as incomplete), takes the lvm
+           snapshot and then updates the status of the in memory and on disk
+           snap objects as complete. Suppose one of the glusterds goes down
+           after taking the lvm snapshot, but before updating the snap object,
+           then treat it as a snapshot create failure and trigger cleanup.
+           i.e the number of commit responses received by the originator
+           glusterd shold be the same as the number of peers it has sent the
+           request to (i.e npeers variable). If not, then originator glusterd
+           will initiate cleanup in post-validate fop.
+           Question: What if one of the other glusterds goes down as explained
+           above and along with it the originator glusterd also goes down?
+           Who will initiate the cleanup?
+        */
+        ret = glusterd_mgmt_v3_commit (conf, op, dict, req_dict,
+                                       &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed");
+                /* If the main op fails, we should save the error string.
+                   Because, op_errstr will be used for unbarrier and
+                   unlock ops also. We might lose the actual error that
+                   caused the failure.
+                */
+                tmp_errstr = op_errstr;
+                op_errstr = NULL;
+                goto unbarrier;
+        }
+
+        success = _gf_true;
+unbarrier:
+        /* BRICK OP PHASE for removing the barrier*/
+        ret = dict_set_int32 (req_dict, "barrier", 0);
+        if (ret)
+                goto out;
+        ret = glusterd_mgmt_v3_brick_op (conf, op, req_dict,
+                                         &op_errstr, npeers);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        op_ret = ret;
+
+        if (success == _gf_false)
+                op_ret = -1;
+
+        /* POST-COMMIT VALIDATE PHASE */
+        ret = glusterd_mgmt_v3_post_validate (conf, op, op_ret, dict, req_dict,
+                                              &op_errstr, npeers);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed");
+                op_ret = -1;
+        }
+
+        /* UNLOCK PHASE FOR PEERS*/
+        (void) glusterd_mgmt_v3_release_peer_locks (conf, op, dict,
+                                                    op_ret, &op_errstr,
+                                                    npeers, is_acquired);
+
+        /* If the commit op (snapshot taking) failed, then the error is stored
+           in tmp_errstr and unbarrier is called. Suppose, if unbarrier also
+           fails, then the error happened in unbarrier is logged and freed.
+           The error happened in commit op, which is stored in tmp_errstr
+           is sent to cli.
+        */
+        if (tmp_errstr) {
+                if (op_errstr) {
+                        gf_log (this->name, GF_LOG_ERROR, "unbarrier brick op"
+                                "failed with the error %s", op_errstr);
+                        GF_FREE (op_errstr);
+                        op_errstr = NULL;
+                }
+                op_errstr = tmp_errstr;
+        }
+
+        /* LOCAL VOLUME(S) UNLOCK */
+        if (is_acquired) {
+                /* Trying to release multiple mgmt_v3 locks */
+                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to release mgmt_v3 locks on localhost");
+                        op_ret = ret;
+                }
+        }
+
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr);
+
+        if (req_dict)
+                dict_unref (req_dict);
+
+        if (tmp_dict)
+                dict_unref (tmp_dict);
+
+        if (op_errstr) {
+                GF_FREE (op_errstr);
+                op_errstr = NULL;
+        }
+
+        return 0;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
new file mode 100644
index 000000000..b185a9bec
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
@@ -0,0 +1,45 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_MGMT_H_
+#define _GLUSTERD_MGMT_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+int32_t
+gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
+                           char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
+                       char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
+                     char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                            char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                     dict_t *dict);
+
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
+                                      dict_t *dict);
+
+int
+glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src);
+
+#endif /* _GLUSTERD_MGMT_H_ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
new file mode 100644
index 000000000..0d67d1303
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
@@ -0,0 +1,693 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <inttypes.h>
+#include <fnmatch.h>
+#include <pwd.h>
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "compat.h"
+#include "dict.h"
+#include "list.h"
+#include "logging.h"
+#include "defaults.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "run.h"
+#include "glusterd-mem-types.h"
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "common-utils.h"
+#include "glusterd-mountbroker.h"
+#include "glusterd-op-sm.h"
+
+static int
+seq_dict_foreach (dict_t *dict,
+                  int (*fn)(char *str, void *data),
+                  void *data)
+{
+        char index[] = "4294967296"; // 1<<32
+        int        i = 0;
+        char    *val = NULL;
+        int      ret = 0;
+
+        for (;;i++) {
+                snprintf(index, sizeof(index), "%d", i);
+                ret = dict_get_str (dict, index, &val);
+                if (ret != 0)
+                        return ret == -ENOENT ? 0 : ret;
+                ret = fn (val, data);
+                if (ret != 0)
+                        return ret;
+        }
+}
+
+int
+parse_mount_pattern_desc (gf_mount_spec_t *mspec, char *pdesc)
+#define SYNTAX_ERR -2
+{
+        char *curs              = NULL;
+        char *c2                = NULL;
+        char sc                 = '\0';
+        char **cc               = NULL;
+        gf_mount_pattern_t *pat = NULL;
+        int pnum                = 0;
+        int ret                 = 0;
+        int lastsup             = -1;
+        int incl                = -1;
+        char **pcc              = NULL;
+        int pnc                 = 0;
+
+        skipwhite (&pdesc);
+
+        /* a bow to theory */
+        if (!*pdesc)
+                return 0;
+
+        /* count number of components, separated by '&' */
+        mspec->len = 0;
+        for (curs = pdesc; *curs; curs++) {
+                if (*curs == ')')
+                        mspec->len++;
+        }
+
+        mspec->patterns = GF_CALLOC (mspec->len, sizeof (*mspec->patterns),
+                                     gf_gld_mt_mount_pattern);
+        if (!mspec->patterns) {
+                ret = -1;
+                goto out;
+        }
+
+        pat = mspec->patterns;
+        curs = pdesc;
+        skipwhite (&curs);
+        for (;;) {
+                incl = -1;
+
+                /* check for pattern signedness modifier */
+                if (*curs == '-') {
+                        pat->negative = _gf_true;
+                        curs++;
+                }
+
+                /* now should come condition specifier,
+                 * then opening paren
+                 */
+                c2 = nwstrtail (curs, "SUB(");
+                if (c2) {
+                        pat->condition = SET_SUB;
+                        goto got_cond;
+                }
+                c2 = nwstrtail (curs, "SUP(");
+                if (c2) {
+                        pat->condition = SET_SUPER;
+                        lastsup = pat - mspec->patterns;
+                        goto got_cond;
+                }
+                c2 = nwstrtail (curs, "EQL(");
+                if (c2) {
+                        pat->condition = SET_EQUAL;
+                        goto got_cond;
+                }
+                c2 = nwstrtail (curs, "MEET(");
+                if (c2) {
+                        pat->condition = SET_INTERSECT;
+                        goto got_cond;
+                }
+                c2 = nwstrtail (curs, "SUB+(");
+                if (c2) {
+                        pat->condition = SET_SUB;
+                        incl = lastsup;
+                        goto got_cond;
+                }
+
+                ret = SYNTAX_ERR;
+                goto out;
+
+ got_cond:
+                curs = c2;
+                skipwhite (&curs);
+                /* count the number of components for pattern */
+                pnum = *curs == ')' ? 0 : 1;
+                for (c2 = curs ;*c2 != ')';) {
+                        if (strchr ("&|", *c2)) {
+                                ret = SYNTAX_ERR;
+                                goto out;
+                        }
+                        while (!strchr ("|&)", *c2) && !isspace (*c2))
+                                c2++;
+                        skipwhite (&c2);
+                        switch (*c2) {
+                        case ')':
+                                break;
+                        case '\0':
+                        case '&':
+                                ret = SYNTAX_ERR;
+                                goto out;
+                        case '|':
+                                *c2 = ' ';
+                                skipwhite (&c2);
+                                /* fall through */
+                        default:
+                                pnum++;
+                        }
+                }
+                if (incl >= 0) {
+                        pnc = 0;
+                        for (pcc = mspec->patterns[incl].components; *pcc; pcc++)
+                                pnc++;
+                        pnum += pnc;
+                }
+                pat->components = GF_CALLOC (pnum + 1, sizeof (*pat->components),
+                                             gf_gld_mt_mount_comp_container);
+                if (!pat->components) {
+                        ret = -1;
+                        goto out;
+                }
+
+                cc = pat->components;
+                /* copy over included component set */
+                if (incl >= 0) {
+                        memcpy (pat->components,
+                                mspec->patterns[incl].components,
+                                pnc * sizeof (*pat->components));
+                        cc += pnc;
+                }
+                /* parse and add components */
+                c2 = ""; /* reset c2 */
+                while (*c2 != ')') {
+                        c2 = curs;
+                        while (!isspace (*c2) && *c2 != ')')
+                                c2++;
+                        sc = *c2;
+                        *c2 = '\0';;
+                        *cc = gf_strdup (curs);
+                        if (!*cc) {
+                                ret = -1;
+                                goto out;
+                        }
+                        *c2 = sc;
+                        skipwhite (&c2);
+                        curs = c2;
+                        cc++;
+                }
+
+                curs++;
+                skipwhite (&curs);
+                if (*curs == '&') {
+                        curs++;
+                        skipwhite (&curs);
+                }
+
+                if (!*curs)
+                        break;
+                pat++;
+        }
+
+ out:
+        if (ret == SYNTAX_ERR) {
+                gf_log ("", GF_LOG_ERROR, "cannot parse mount patterns %s",
+                        pdesc);
+        }
+
+        /* We've allocted a lotta stuff here but don't bother with freeing
+         * on error, in that case we'll terminate anyway
+         */
+        return ret ? -1 : 0;
+}
+#undef SYNTAX_ERR
+
+
+const char *georep_mnt_desc_template =
+        "SUP("
+                "xlator-option=\\*-dht.assert-no-child-down=true "
+                "volfile-server=localhost "
+                "client-pid=%d "
+                "user-map-root=%s "
+        ")"
+        "SUB+("
+                "log-file="DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"*/* "
+                "log-level=* "
+                "volfile-id=* "
+        ")"
+        "MEET("
+                "%s"
+        ")";
+
+const char *hadoop_mnt_desc_template =
+        "SUP("
+                "volfile-server=%s "
+                "client-pid=%d "
+                "volfile-id=%s "
+                "user-map-root=%s "
+        ")"
+        "SUB+("
+                "log-file="DEFAULT_LOG_FILE_DIRECTORY"/"GHADOOP"*/* "
+                "log-level=* "
+        ")";
+
+int
+make_georep_mountspec (gf_mount_spec_t *mspec, const char *volnames,
+                       char *user)
+{
+        char *georep_mnt_desc = NULL;
+        char *meetspec        = NULL;
+        char *vols            = NULL;
+        char *vol             = NULL;
+        char *p               = NULL;
+        char *savetok         = NULL;
+        char *fa[3]           = {0,};
+        size_t siz            = 0;
+        int vc                = 0;
+        int i                 = 0;
+        int ret               = 0;
+
+        vols = gf_strdup ((char *)volnames);
+        if (!vols)
+                goto out;
+
+        for (vc = 1, p = vols; *p; p++) {
+                if (*p == ',')
+                        vc++;
+        }
+        siz = strlen (volnames) + vc * strlen("volfile-id=");
+        meetspec = GF_CALLOC (1, siz + 1, gf_gld_mt_georep_meet_spec);
+        if (!meetspec)
+                goto out;
+
+        for (p = vols;;) {
+                vol = strtok_r (p, ",", &savetok);
+                if (!vol) {
+                        GF_ASSERT (vc == 0);
+                        break;
+                }
+                p = NULL;
+                strcat (meetspec, "volfile-id=");
+                strcat (meetspec, vol);
+                if (--vc > 0)
+                        strcat (meetspec, " ");
+        }
+
+        ret = gf_asprintf (&georep_mnt_desc, georep_mnt_desc_template,
+                           GF_CLIENT_PID_GSYNCD, user, meetspec);
+        if (ret == -1) {
+                georep_mnt_desc = NULL;
+                goto out;
+        }
+
+        ret = parse_mount_pattern_desc (mspec, georep_mnt_desc);
+
+ out:
+        fa[0] = meetspec;
+        fa[1] = vols;
+        fa[2] = georep_mnt_desc;
+
+        for (i = 0; i < 3; i++) {
+                if (fa[i] == NULL)
+                        ret = -1;
+                else
+                        GF_FREE (fa[i]);
+        }
+
+        return ret;
+}
+
+int
+make_ghadoop_mountspec (gf_mount_spec_t *mspec, const char *volname,
+                        char *user, char *server)
+{
+        char *hadoop_mnt_desc = NULL;
+        int   ret             = 0;
+
+        ret = gf_asprintf (&hadoop_mnt_desc, hadoop_mnt_desc_template,
+                           server, GF_CLIENT_PID_HADOOP, volname, user);
+        if (ret == -1)
+                return ret;
+
+        return parse_mount_pattern_desc (mspec, hadoop_mnt_desc);
+}
+
+static gf_boolean_t
+match_comp (char *str, char *patcomp)
+{
+        char *c1 = patcomp;
+        char *c2 = str;
+
+        GF_ASSERT (c1);
+        GF_ASSERT (c2);
+
+        while (*c1 == *c2) {
+                if (!*c1)
+                        return _gf_true;
+                c1++;
+                c2++;
+                if (c1[-1] == '=')
+                        break;
+        }
+
+        return fnmatch (c1, c2, 0) == 0 ? _gf_true : _gf_false;
+}
+
+struct gf_set_descriptor {
+        gf_boolean_t priv[2];
+        gf_boolean_t common;
+};
+
+static int
+_gf_set_dict_iter1 (char *val, void *data)
+{
+        void **dataa                 = data;
+        struct gf_set_descriptor *sd = dataa[0];
+        char **curs                  = dataa[1];
+        gf_boolean_t priv            = _gf_true;
+
+        while (*curs) {
+                if (match_comp (val, *curs)) {
+                        priv = _gf_false;
+                        sd->common = _gf_true;
+                }
+                curs++;
+        }
+
+        if (priv)
+                sd->priv[0] = _gf_true;
+
+        return 0;
+}
+
+static int
+_gf_set_dict_iter2 (char *val, void *data)
+{
+        void **dataa      = data;
+        gf_boolean_t *boo = dataa[0];
+        char *comp        = dataa[1];
+
+        if (match_comp (val, comp))
+                *boo = _gf_true;
+
+        return 0;
+}
+
+static void
+relate_sets (struct gf_set_descriptor *sd, dict_t *argdict, char **complist)
+{
+        void *dataa[] = {NULL, NULL};
+        gf_boolean_t boo = _gf_false;
+
+        memset (sd, 0, sizeof (*sd));
+
+        dataa[0] = sd;
+        dataa[1] = complist;
+        seq_dict_foreach (argdict, _gf_set_dict_iter1, dataa);
+
+        while (*complist) {
+                boo = _gf_false;
+                dataa[0] = &boo;
+                dataa[1] = *complist;
+                seq_dict_foreach (argdict, _gf_set_dict_iter2, dataa);
+
+                if (boo)
+                        sd->common = _gf_true;
+                else
+                        sd->priv[1] = _gf_true;
+
+                complist++;
+        }
+}
+
+static int
+_arg_parse_uid (char *val, void *data)
+{
+        char *user        = strtail (val, "user-map-root=");
+        struct passwd *pw = NULL;
+
+        if (!user)
+                return 0;
+        pw = getpwnam (user);
+        if (!pw)
+                return -EINVAL;
+
+        if (*(int *)data >= 0)
+                /* uid ambiguity, already found */
+                return -EINVAL;
+
+        *(int *)data = pw->pw_uid;
+        return 0;
+}
+
+static int
+evaluate_mount_request (gf_mount_spec_t *mspec, dict_t *argdict)
+{
+        struct gf_set_descriptor sd = {{0,},};
+        int i                       = 0;
+        int uid                     = -1;
+        int ret                     = 0;
+        gf_boolean_t match          = _gf_false;
+
+        for (i = 0; i < mspec->len; i++) {
+                relate_sets (&sd, argdict, mspec->patterns[i].components);
+                switch (mspec->patterns[i].condition) {
+                case SET_SUB:
+                        match = !sd.priv[0];
+                        break;
+                case SET_SUPER:
+                        match = !sd.priv[1];
+                        break;
+                case SET_EQUAL:
+                        match = (!sd.priv[0] && !sd.priv[1]);
+                        break;
+                case SET_INTERSECT:
+                        match = sd.common;
+                        break;
+                default:
+                        GF_ASSERT(!"unreached");
+                }
+                if (mspec->patterns[i].negative)
+                        match = !match;
+
+                if (!match)
+                        return -EPERM;
+        }
+
+        ret = seq_dict_foreach (argdict, _arg_parse_uid, &uid);
+        if (ret != 0)
+                return ret;
+
+        return uid;
+}
+
+static int
+_volname_get (char *val, void *data)
+{
+        char **volname = data;
+
+        *volname = strtail (val, "volfile-id=");
+
+        return *volname ? 1 : 0;
+}
+
+static int
+_runner_add (char *val, void *data)
+{
+        runner_t *runner = data;
+
+        runner_argprintf (runner, "--%s", val);
+
+        return 0;
+}
+
+int
+glusterd_do_mount (char *label, dict_t *argdict, char **path, int *op_errno)
+{
+        glusterd_conf_t *priv      = NULL;
+        char *mountbroker_root     = NULL;
+        gf_mount_spec_t *mspec     = NULL;
+        int uid                    = -ENOENT;
+        char *volname              = NULL;
+        glusterd_volinfo_t *vol    = NULL;
+        char *mtptemp              = NULL;
+        char *mntlink              = NULL;
+        char *cookieswitch         = NULL;
+        char *cookie               = NULL;
+        char *sla                  = NULL;
+        struct stat st             = {0,};
+        runner_t runner            = {0,};
+        int ret                    = 0;
+        xlator_t *this             = THIS;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (op_errno);
+        *op_errno = 0;
+
+        if (dict_get_str (this->options, "mountbroker-root",
+                          &mountbroker_root) != 0) {
+                *op_errno = ENOENT;
+                goto out;
+        }
+
+        GF_ASSERT (label);
+        if (!*label) {
+                *op_errno = EINVAL;
+                goto out;
+        }
+
+        /* look up spec for label */
+        list_for_each_entry (mspec, &priv->mount_specs,
+                             speclist) {
+                if (strcmp (mspec->label, label) != 0)
+                        continue;
+                uid = evaluate_mount_request (mspec, argdict);
+                break;
+        }
+        if (uid < 0) {
+                *op_errno = -uid;
+                goto out;
+        }
+
+        /* some sanity check on arguments */
+        seq_dict_foreach (argdict, _volname_get, &volname);
+        if (!volname) {
+                *op_errno = EINVAL;
+                goto out;
+        }
+        if (glusterd_volinfo_find (volname, &vol) != 0 ||
+            !glusterd_is_volume_started (vol)) {
+                *op_errno = ENOENT;
+                goto out;
+        }
+
+        /* go do mount */
+
+        /** create actual mount dir */
+
+        /*** "overload" string name to be possible to used for cookie
+             creation, see below */
+        ret = gf_asprintf (&mtptemp, "%s/user%d/mtpt-%s-XXXXXX/cookie",
+                           mountbroker_root, uid, label);
+        if (ret == -1) {
+                mtptemp = NULL;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+        /*** hide cookie part */
+        cookieswitch = strrchr (mtptemp, '/');
+        *cookieswitch = '\0';
+
+        sla = strrchr (mtptemp, '/');
+        *sla = '\0';
+        ret = mkdir (mtptemp, 0700);
+        if (ret == 0)
+                ret = chown (mtptemp, uid, 0);
+        else if (errno == EEXIST)
+                ret = 0;
+        if (ret == -1) {
+                *op_errno = errno;
+                goto out;
+        }
+        ret = lstat (mtptemp, &st);
+        if (ret == -1) {
+                *op_errno = errno;
+                goto out;
+        }
+        if (!(S_ISDIR (st.st_mode) && (st.st_mode & ~S_IFMT) == 0700 &&
+              st.st_uid == uid && st.st_gid == 0)) {
+                *op_errno = EACCES;
+                goto out;
+        }
+        *sla = '/';
+
+        if (!mkdtemp (mtptemp)) {
+                *op_errno = errno;
+                goto out;
+        }
+
+        /** create private "cookie" symlink */
+
+        /*** occupy an entry in the hive dir via mkstemp */
+        ret = gf_asprintf (&cookie, "%s/"MB_HIVE"/mntXXXXXX",
+                           mountbroker_root);
+        if (ret == -1) {
+                cookie = NULL;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+        ret = mkstemp (cookie);
+        if (ret == -1) {
+                *op_errno = errno;
+                goto out;
+        }
+        close (ret);
+
+        /*** assembly the path from cookie to mountpoint */
+        sla = strchr (sla - 1, '/');
+        GF_ASSERT (sla);
+        ret = gf_asprintf (&mntlink, "../user%d%s", uid, sla);
+        if (ret == -1) {
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        /*** create cookie link in (to-be) mountpoint,
+             move it over to the final place */
+        *cookieswitch = '/';
+        ret = symlink (mntlink, mtptemp);
+        if (ret != -1)
+                ret = rename (mtptemp, cookie);
+        *cookieswitch = '\0';
+        if (ret == -1) {
+                *op_errno = errno;
+                goto out;
+        }
+
+        /** invoke glusterfs on the mountpoint */
+
+        runinit (&runner);
+        runner_add_arg (&runner, SBIN_DIR"/glusterfs");
+        seq_dict_foreach (argdict, _runner_add, &runner);
+        runner_add_arg (&runner, mtptemp);
+        ret = runner_run_reuse (&runner);
+        if (ret == -1) {
+                *op_errno = EIO; /* XXX hacky fake */
+                runner_log (&runner, "", GF_LOG_ERROR, "command failed");
+        }
+        runner_end (&runner);
+
+ out:
+
+        if (*op_errno) {
+                ret = -1;
+                gf_log ("", GF_LOG_WARNING, "unsuccessful mount request (%s)",
+                        strerror (*op_errno));
+                if (mtptemp) {
+                        *cookieswitch = '/';
+                        unlink (mtptemp);
+                        *cookieswitch = '\0';
+                        rmdir (mtptemp);
+                }
+                if (cookie) {
+                        unlink (cookie);
+                        GF_FREE (cookie);
+                }
+
+        } else {
+                ret = 0;
+                *path = cookie;
+        }
+
+        GF_FREE (mtptemp);
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.h b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
new file mode 100644
index 000000000..426252ebe
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
@@ -0,0 +1,42 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#define MB_HIVE "mb_hive"
+
+typedef enum {
+        SET_SUB = 1,
+        SET_SUPER,
+        SET_EQUAL,
+        SET_INTERSECT
+} gf_setrel_t;
+
+struct gf_mount_pattern {
+        char **components;
+        gf_setrel_t condition;
+        gf_boolean_t negative;
+};
+typedef struct gf_mount_pattern gf_mount_pattern_t;
+
+struct gf_mount_spec {
+        struct list_head speclist;
+        char *label;
+        gf_mount_pattern_t *patterns;
+        size_t len;
+};
+typedef struct gf_mount_spec gf_mount_spec_t;
+
+
+int parse_mount_pattern_desc (gf_mount_spec_t *mspec, char *pdesc);
+
+int make_georep_mountspec (gf_mount_spec_t *mspec, const char *volname,
+                           char *user);
+int make_ghadoop_mountspec (gf_mount_spec_t *mspec, const char *volname,
+                            char *user, char *server);
+
+int glusterd_do_mount (char *label, dict_t *argdict, char **path, int *op_errno);
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
new file mode 100644
index 000000000..1666f5e4d
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -0,0 +1,6243 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <time.h>
+#include <sys/uio.h>
+#include <sys/resource.h>
+#include <sys/mount.h>
+
+#include <libgen.h>
+#include "uuid.h"
+
+#include "fnmatch.h"
+#include "xlator.h"
+#include "protocol-common.h"
+#include "glusterd.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "list.h"
+#include "dict.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "statedump.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "glusterd-hooks.h"
+#include "glusterd-volgen.h"
+#include "glusterd-locks.h"
+#include "syscall.h"
+#include "cli1-xdr.h"
+#include "common-utils.h"
+#include "run.h"
+
+#include <sys/types.h>
+#include <signal.h>
+#include <sys/wait.h>
+
+#define ALL_VOLUME_OPTION_CHECK(volname, key, ret, op_errstr, label)           \
+        do {                                                                   \
+                gf_boolean_t    _all = !strcmp ("all", volname);               \
+                gf_boolean_t    _ratio = !strcmp (key,                         \
+                                                  GLUSTERD_QUORUM_RATIO_KEY);  \
+                if (_all && !_ratio) {                                         \
+                        ret = -1;                                              \
+                        *op_errstr = gf_strdup ("Not a valid option for all "  \
+                                                "volumes");                    \
+                        goto label;                                            \
+                } else if (!_all && _ratio) {                                  \
+                        ret = -1;                                              \
+                        *op_errstr = gf_strdup ("Not a valid option for "      \
+                                                "single volume");              \
+                        goto label;                                            \
+                }                                                              \
+         } while (0)
+
+static struct list_head gd_op_sm_queue;
+pthread_mutex_t       gd_op_sm_lock;
+glusterd_op_info_t    opinfo = {{0},};
+uuid_t                global_txn_id = {"\0"}; /* To be used in
+                                               * heterogeneous
+                                               * cluster with no
+                                               * transaction ids */
+
+static dict_t *txn_opinfo;
+
+struct txn_opinfo_object_ {
+        glusterd_op_info_t    opinfo;
+};
+typedef struct txn_opinfo_object_ txn_opinfo_obj;
+
+int32_t
+glusterd_txn_opinfo_dict_init ()
+{
+        int32_t ret = -1;
+
+        txn_opinfo = dict_new ();
+        if (!txn_opinfo) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+glusterd_txn_opinfo_dict_fini ()
+{
+        if (txn_opinfo)
+                dict_destroy (txn_opinfo);
+}
+
+void
+glusterd_txn_opinfo_init (glusterd_op_info_t  *opinfo,
+                          glusterd_op_sm_state_info_t *state,
+                          glusterd_op_t *op,
+                          dict_t *op_ctx,
+                          rpcsvc_request_t *req)
+{
+        GF_ASSERT (opinfo);
+
+        if (state)
+                opinfo->state = *state;
+
+        if (op)
+                opinfo->op = *op;
+
+        opinfo->op_ctx = dict_ref(op_ctx);
+
+        if (req)
+                opinfo->req = req;
+
+        return;
+}
+
+int32_t
+glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+{
+        int32_t          ret         = -1;
+        txn_opinfo_obj  *opinfo_obj  = NULL;
+
+        if (!txn_id || !opinfo) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Empty transaction id or opinfo received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(txn_opinfo, uuid_utoa (*txn_id),
+                           (void **) &opinfo_obj);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to get transaction opinfo");
+                goto out;
+        }
+
+        (*opinfo) = opinfo_obj->opinfo;
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+{
+        int32_t          ret        = -1;
+        txn_opinfo_obj  *opinfo_obj = NULL;
+
+        if (!txn_id) {
+                gf_log ("", GF_LOG_ERROR, "Empty transaction id received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_bin(txn_opinfo, uuid_utoa (*txn_id),
+                           (void **) &opinfo_obj);
+        if (ret) {
+                opinfo_obj = GF_CALLOC (1, sizeof(txn_opinfo_obj),
+                                        gf_common_mt_txn_opinfo_obj_t);
+                if (!opinfo_obj) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_set_bin(txn_opinfo, uuid_utoa (*txn_id), opinfo_obj,
+                                   sizeof(txn_opinfo_obj));
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set opinfo for transaction ID : %s",
+                                uuid_utoa (*txn_id));
+                        goto out;
+                }
+        }
+
+        opinfo_obj->opinfo = (*opinfo);
+
+        ret = 0;
+out:
+        if (ret)
+                if (opinfo_obj)
+                        GF_FREE (opinfo_obj);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_clear_txn_opinfo (uuid_t *txn_id)
+{
+        int32_t               ret         = -1;
+        glusterd_op_info_t    txn_op_info = {{0},};
+
+        if (!txn_id) {
+                gf_log ("", GF_LOG_ERROR, "Empty transaction id received.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Transaction opinfo not found");
+                goto out;
+        }
+
+        dict_unref (txn_op_info.op_ctx);
+
+        dict_del(txn_opinfo, uuid_utoa (*txn_id));
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int glusterfs_port = GLUSTERD_DEFAULT_PORT;
+static char *glusterd_op_sm_state_names[] = {
+        "Default",
+        "Lock sent",
+        "Locked",
+        "Stage op sent",
+        "Staged",
+        "Commit op sent",
+        "Committed",
+        "Unlock sent",
+        "Stage op failed",
+        "Commit op failed",
+        "Brick op sent",
+        "Brick op failed",
+        "Brick op Committed",
+        "Brick op Commit failed",
+        "Ack drain",
+        "Invalid",
+};
+
+static char *glusterd_op_sm_event_names[] = {
+        "GD_OP_EVENT_NONE",
+        "GD_OP_EVENT_START_LOCK",
+        "GD_OP_EVENT_LOCK",
+        "GD_OP_EVENT_RCVD_ACC",
+        "GD_OP_EVENT_ALL_ACC",
+        "GD_OP_EVENT_STAGE_ACC",
+        "GD_OP_EVENT_COMMIT_ACC",
+        "GD_OP_EVENT_RCVD_RJT",
+        "GD_OP_EVENT_STAGE_OP",
+        "GD_OP_EVENT_COMMIT_OP",
+        "GD_OP_EVENT_UNLOCK",
+        "GD_OP_EVENT_START_UNLOCK",
+        "GD_OP_EVENT_ALL_ACK",
+        "GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP",
+        "GD_OP_EVENT_INVALID"
+};
+
+extern struct volopt_map_entry glusterd_volopt_map[];
+
+char*
+glusterd_op_sm_state_name_get (int state)
+{
+        if (state < 0 || state >= GD_OP_STATE_MAX)
+                return glusterd_op_sm_state_names[GD_OP_STATE_MAX];
+        return glusterd_op_sm_state_names[state];
+}
+
+char*
+glusterd_op_sm_event_name_get (int event)
+{
+        if (event < 0 || event >= GD_OP_EVENT_MAX)
+                return glusterd_op_sm_event_names[GD_OP_EVENT_MAX];
+        return glusterd_op_sm_event_names[event];
+}
+
+void
+glusterd_destroy_lock_ctx (glusterd_op_lock_ctx_t *ctx)
+{
+        if (!ctx)
+                return;
+        GF_FREE (ctx);
+}
+
+void
+glusterd_set_volume_status (glusterd_volinfo_t  *volinfo,
+                            glusterd_volume_status status)
+{
+        GF_ASSERT (volinfo);
+        volinfo->status = status;
+}
+
+gf_boolean_t
+glusterd_is_volume_started (glusterd_volinfo_t  *volinfo)
+{
+        GF_ASSERT (volinfo);
+        return (volinfo->status == GLUSTERD_STATUS_STARTED);
+}
+
+static int
+glusterd_op_sm_inject_all_acc (uuid_t *txn_id)
+{
+        int32_t                 ret = -1;
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, txn_id, NULL);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickinfo,
+                                 gd1_mgmt_brick_op_req **req, dict_t *dict)
+{
+        int                     ret = -1;
+        gd1_mgmt_brick_op_req   *brick_req = NULL;
+        char                    *volname = NULL;
+        char                    name[1024] = {0,};
+        gf_xl_afr_op_t          heal_op = GF_AFR_OP_INVALID;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (op < GD_OP_MAX);
+        GF_ASSERT (op > GD_OP_NONE);
+        GF_ASSERT (req);
+
+
+        switch (op) {
+        case GD_OP_REMOVE_BRICK:
+        case GD_OP_STOP_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+                brick_req->op = GLUSTERD_BRICK_TERMINATE;
+                brick_req->name = "";
+        break;
+        case GD_OP_PROFILE_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_BRICK_XLATOR_INFO;
+                brick_req->name = brickinfo->path;
+
+                break;
+        case GD_OP_HEAL_VOLUME:
+        {
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_BRICK_XLATOR_OP;
+                brick_req->name = "";
+                ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
+                if (ret)
+                        goto out;
+                ret = dict_set_int32 (dict, "xl-op", heal_op);
+        }
+                break;
+        case GD_OP_STATUS_VOLUME:
+        {
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+                brick_req->op = GLUSTERD_BRICK_STATUS;
+                brick_req->name = "";
+        }
+                break;
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_BRICK_XLATOR_DEFRAG;
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto out;
+                snprintf (name, 1024, "%s-dht",volname);
+                brick_req->name = gf_strdup (name);
+
+                break;
+        case GD_OP_SNAP:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_VOLUME_BARRIER_OP;
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret)
+                        goto out;
+                snprintf (name, 1024, "%s-server",volname);
+                brick_req->name = gf_strdup (name);
+
+                break;
+        default:
+                goto out;
+        break;
+        }
+
+        ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
+                                           &brick_req->input.input_len);
+        if (ret)
+                goto out;
+        *req = brick_req;
+        ret = 0;
+
+out:
+        if (ret && brick_req)
+                GF_FREE (brick_req);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+                                dict_t *dict)
+{
+        int                     ret = -1;
+        gd1_mgmt_brick_op_req   *brick_req = NULL;
+
+        GF_ASSERT (op < GD_OP_MAX);
+        GF_ASSERT (op > GD_OP_NONE);
+        GF_ASSERT (req);
+
+        switch (op) {
+        case GD_OP_PROFILE_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_NODE_PROFILE;
+                brick_req->name = "";
+
+                break;
+
+        case GD_OP_STATUS_VOLUME:
+                brick_req = GF_CALLOC (1, sizeof (*brick_req),
+                                       gf_gld_mt_mop_brick_req_t);
+                if (!brick_req)
+                        goto out;
+
+                brick_req->op = GLUSTERD_NODE_STATUS;
+                brick_req->name = "";
+
+                break;
+
+        default:
+                goto out;
+        }
+
+        ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
+                                           &brick_req->input.input_len);
+
+        if (ret)
+                goto out;
+
+        *req = brick_req;
+        ret = 0;
+
+out:
+        if (ret && brick_req)
+                GF_FREE (brick_req);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_validate_quorum_options (xlator_t *this, char *fullkey, char *value,
+                                  char **op_errstr)
+{
+        int             ret = 0;
+        char            *key = NULL;
+        volume_option_t *opt = NULL;
+
+        if (!glusterd_is_quorum_option (fullkey))
+                goto out;
+        key = strchr (fullkey, '.');
+        key++;
+        opt = xlator_volume_option_get (this, key);
+        ret = xlator_option_validate (this, key, value, opt, op_errstr);
+out:
+        return ret;
+}
+
+static int
+glusterd_check_client_op_version_support (char *volname, uint32_t op_version,
+                                          char **op_errstr)
+{
+        int                     ret = 0;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+        rpc_transport_t         *xprt = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        priv = this->private;
+        GF_ASSERT(priv);
+
+        pthread_mutex_lock (&priv->xprt_lock);
+        list_for_each_entry (xprt, &priv->xprt_list, list) {
+                if ((!strcmp(volname, xprt->peerinfo.volname)) &&
+                    ((op_version > xprt->peerinfo.max_op_version) ||
+                     (op_version < xprt->peerinfo.min_op_version))) {
+                        ret = -1;
+                        break;
+                }
+        }
+        pthread_mutex_unlock (&priv->xprt_lock);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "One or more clients "
+                        "don't support the required op-version");
+                ret = gf_asprintf (op_errstr, "One or more connected clients "
+                                   "cannot support the feature being set. "
+                                   "These clients need to be upgraded or "
+                                   "disconnected before running this command"
+                                   " again");
+                return -1;
+        }
+        return 0;
+}
+
+static int
+glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
+{
+        int                             ret                     = -1;
+        char                            *volname                = NULL;
+        int                             exists                  = 0;
+        char                            *key                    = NULL;
+        char                            *key_fixed              = NULL;
+        char                            *value                  = NULL;
+        char                            str[100]                = {0, };
+        int                             count                   = 0;
+        int                             dict_count              = 0;
+        char                            errstr[2048]            = {0, };
+        glusterd_volinfo_t              *volinfo                = NULL;
+        dict_t                          *val_dict               = NULL;
+        gf_boolean_t                    global_opt              = _gf_false;
+        glusterd_volinfo_t              *voliter                = NULL;
+        glusterd_conf_t                 *priv                   = NULL;
+        xlator_t                        *this                   = NULL;
+        uint32_t                        new_op_version          = 0;
+        uint32_t                        local_new_op_version    = 0;
+        uint32_t                        key_op_version          = 0;
+        uint32_t                        local_key_op_version    = 0;
+        gf_boolean_t                    origin_glusterd         = _gf_true;
+        gf_boolean_t                    check_op_version        = _gf_true;
+        gf_boolean_t                    all_vol                 = _gf_false;
+        struct volopt_map_entry         *vme                    = NULL;
+
+        GF_ASSERT (dict);
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        val_dict = dict_new();
+        if (!val_dict)
+                goto out;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        /* Check if we can support the required op-version
+         * This check is not done on the originator glusterd. The originator
+         * glusterd sets this value.
+         */
+        origin_glusterd = is_origin_glusterd (dict);
+
+        if (!origin_glusterd) {
+                /* Check for v3.3.x origin glusterd */
+                check_op_version = dict_get_str_boolean (dict,
+                                                         "check-op-version",
+                                                         _gf_false);
+
+                if (check_op_version) {
+                        ret = dict_get_uint32 (dict, "new-op-version",
+                                               &new_op_version);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get new_op_version");
+                                goto out;
+                        }
+
+                        if ((new_op_version > GD_OP_VERSION_MAX) ||
+                            (new_op_version < GD_OP_VERSION_MIN)) {
+                                ret = -1;
+                                snprintf (errstr, sizeof (errstr),
+                                          "Required op_version (%d) is not "
+                                          "supported", new_op_version);
+                                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                                goto out;
+                        }
+                }
+        }
+
+        ret = dict_get_int32 (dict, "count", &dict_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Count(dict),not set in Volume-Set");
+                goto out;
+        }
+
+        if (dict_count == 0) {
+        /*No options would be specified of volume set help */
+                if (dict_get (dict, "help" ))  {
+                        ret = 0;
+                        goto out;
+                }
+
+                if (dict_get (dict, "help-xml" )) {
+#if (HAVE_LIB_XML)
+                        ret = 0;
+                        goto out;
+#else
+                        ret  = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "libxml not present in the system");
+                        *op_errstr = gf_strdup ("Error: xml libraries not "
+                                                "present to produce xml-output");
+                        goto out;
+#endif
+                }
+                gf_log (this->name, GF_LOG_ERROR, "No options received ");
+                *op_errstr = gf_strdup ("Options not specified");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        if (strcasecmp (volname, "all") != 0) {
+                exists = glusterd_check_volume_exists (volname);
+                if (!exists) {
+                        snprintf (errstr, sizeof (errstr),
+                                  FMTSTR_CHECK_VOL_EXISTS, volname);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                FMTSTR_CHECK_VOL_EXISTS, volname);
+                        goto out;
+                }
+
+                ret = glusterd_validate_volume_id (dict, volinfo);
+                if (ret)
+                        goto out;
+        } else {
+                all_vol = _gf_true;
+        }
+
+        local_new_op_version = priv->op_version;
+
+        for ( count = 1; ret != 1 ; count++ ) {
+                global_opt = _gf_false;
+                sprintf (str, "key%d", count);
+                ret = dict_get_str (dict, str, &key);
+                if (ret)
+                        break;
+
+                sprintf (str, "value%d", count);
+                ret = dict_get_str (dict, str, &value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid key,value pair in 'volume set'");
+                        ret = -1;
+                        goto out;
+                }
+
+                if (strcmp (key, "config.memory-accounting") == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "enabling memory accounting for volume %s",
+                                volname);
+                        ret = 0;
+                }
+
+                if (strcmp (key, "config.transport") == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "changing transport-type for volume %s",
+                                volname);
+                        ret = 0;
+                        /* if value is none of 'tcp/rdma/tcp,rdma' error out */
+                        if (!((strcasecmp (value, "rdma") == 0) ||
+                              (strcasecmp (value, "tcp") == 0) ||
+                              (strcasecmp (value, "tcp,rdma") == 0) ||
+                              (strcasecmp (value, "rdma,tcp") == 0))) {
+                                ret = snprintf (errstr, sizeof (errstr),
+                                                "transport-type %s does "
+                                                "not exist", value);
+                                /* lets not bother about above return value,
+                                   its a failure anyways */
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                if (is_key_glusterd_hooks_friendly (key))
+                        continue;
+
+                for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
+                        if ((vme->validate_fn) &&
+                            ((!strcmp (key, vme->key)) ||
+                             (!strcmp (key, strchr (vme->key, '.') + 1)))) {
+                                ret = vme->validate_fn (dict, key, value,
+                                                        op_errstr);
+                                if (ret)
+                                        goto out;
+                                break;
+                        }
+                }
+
+                exists = glusterd_check_option_exists (key, &key_fixed);
+                if (exists == -1) {
+                        ret = -1;
+                        goto out;
+                }
+
+                if (!exists) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Option with name: %s does not exist", key);
+                        ret = snprintf (errstr, sizeof (errstr),
+                                       "option : %s does not exist",
+                                       key);
+                        if (key_fixed)
+                                snprintf (errstr + ret, sizeof (errstr) - ret,
+                                          "\nDid you mean %s?", key_fixed);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (key_fixed)
+                        key = key_fixed;
+                ALL_VOLUME_OPTION_CHECK (volname, key, ret, op_errstr, out);
+                ret = glusterd_validate_quorum_options (this, key, value,
+                                                        op_errstr);
+                if (ret)
+                        goto out;
+
+                local_key_op_version = glusterd_get_op_version_for_key (key);
+                if (local_key_op_version > local_new_op_version)
+                        local_new_op_version = local_key_op_version;
+
+                sprintf (str, "op-version%d", count);
+                if (origin_glusterd) {
+                        ret = dict_set_uint32 (dict, str, local_key_op_version);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set key-op-version in dict");
+                                goto out;
+                        }
+                } else if (check_op_version) {
+                        ret = dict_get_uint32 (dict, str, &key_op_version);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get key-op-version from"
+                                        " dict");
+                                goto out;
+                        }
+                        if (local_key_op_version != key_op_version) {
+                                ret = -1;
+                                snprintf (errstr, sizeof (errstr),
+                                          "option: %s op-version mismatch",
+                                          key);
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "%s, required op-version = %"PRIu32", "
+                                        "available op-version = %"PRIu32,
+                                        errstr, key_op_version,
+                                        local_key_op_version);
+                                goto out;
+                        }
+                }
+
+                if (glusterd_check_globaloption (key))
+                        global_opt = _gf_true;
+
+                ret = dict_set_str (val_dict, key, value);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set the options in 'volume set'");
+                        ret = -1;
+                        goto out;
+                }
+
+                *op_errstr = NULL;
+                if (!global_opt && !all_vol)
+                        ret = glusterd_validate_reconfopts (volinfo, val_dict, op_errstr);
+                else if (!all_vol) {
+                        voliter = NULL;
+                        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                                ret = glusterd_validate_globalopts (voliter, val_dict, op_errstr);
+                                if (ret)
+                                        break;
+                        }
+                }
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not create "
+                                "temp volfile, some option failed: %s",
+                                *op_errstr);
+                        goto out;
+                }
+                dict_del (val_dict, key);
+
+                if (key_fixed) {
+                        GF_FREE (key_fixed);
+                        key_fixed = NULL;
+                }
+        }
+
+        // Check if all the connected clients support the new op-version
+        ret = glusterd_check_client_op_version_support (volname,
+                                                        local_new_op_version,
+                                                        op_errstr);
+        if (ret)
+                goto out;
+
+        if (origin_glusterd) {
+                ret = dict_set_uint32 (dict, "new-op-version",
+                                       local_new_op_version);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set new-op-version in dict");
+                        goto out;
+                }
+                /* Set this value in dict so other peers know to check for
+                 * op-version. This is a hack for 3.3.x compatibility
+                 *
+                 * TODO: Remove this and the other places this is referred once
+                 * 3.3.x compatibility is not required
+                 */
+                ret = dict_set_uint32 (dict, "check-op-version",
+                                       _gf_true);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set check-op-version in dict");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+
+out:
+        if (val_dict)
+                dict_unref (val_dict);
+
+        GF_FREE (key_fixed);
+        if (errstr[0] != '\0')
+                *op_errstr = gf_strdup (errstr);
+
+        if (ret) {
+                if (!(*op_errstr)) {
+                        *op_errstr = gf_strdup ("Error, Validation Failed");
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Error, Cannot Validate option :%s",
+                                *op_errstr);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Error, Cannot Validate option");
+                }
+        }
+        return ret;
+}
+
+static int
+glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
+{
+        int                                      ret           = 0;
+        char                                    *volname       = NULL;
+        gf_boolean_t                             exists        = _gf_false;
+        char                                    msg[2048]      = {0};
+        char                                    *key = NULL;
+        char                                    *key_fixed = NULL;
+        glusterd_volinfo_t                      *volinfo       = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        if (strcasecmp (volname, "all") != 0) {
+                exists = glusterd_check_volume_exists (volname);
+                if (!exists) {
+                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
+                                  volname);
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
+                                  volname);
+                        goto out;
+                }
+
+                ret = glusterd_validate_volume_id (dict, volinfo);
+                if (ret)
+                        goto out;
+        }
+
+        ret = dict_get_str (dict, "key", &key);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get option key");
+                goto out;
+        }
+        if (strcmp(key, "all")) {
+                exists = glusterd_check_option_exists (key, &key_fixed);
+                if (exists == -1) {
+                        ret = -1;
+                        goto out;
+                }
+                if (!exists) {
+                        ret = snprintf (msg, sizeof (msg),
+                                        "Option %s does not exist", key);
+                        if (key_fixed)
+                                snprintf (msg + ret, sizeof (msg) - ret,
+                                          "\nDid you mean %s?", key_fixed);
+                        ret = -1;
+                        goto out;
+                } else if (exists > 0) {
+                        if (key_fixed)
+                                key = key_fixed;
+                        ALL_VOLUME_OPTION_CHECK (volname, key, ret,
+                                                 op_errstr, out);
+                }
+        }
+
+out:
+        GF_FREE (key_fixed);
+
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+
+
+static int
+glusterd_op_stage_sync_volume (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        char                                    *hostname = NULL;
+        gf_boolean_t                            exists = _gf_false;
+        glusterd_peerinfo_t                     *peerinfo = NULL;
+        char                                    msg[2048] = {0,};
+        glusterd_volinfo_t                      *volinfo  = NULL;
+
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "hostname couldn't be "
+                          "retrieved from msg");
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        if (gf_is_local_addr (hostname)) {
+                //volname is not present in case of sync all
+                ret = dict_get_str (dict, "volname", &volname);
+                if (!ret) {
+                        exists = glusterd_check_volume_exists (volname);
+                        if (!exists) {
+                                snprintf (msg, sizeof (msg), "Volume %s "
+                                         "does not exist", volname);
+                                *op_errstr = gf_strdup (msg);
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret)
+                                goto out;
+
+                } else {
+                        ret = 0;
+                }
+         } else {
+                ret = glusterd_friend_find (NULL, hostname, &peerinfo);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), "%s, is not a friend",
+                                  hostname);
+                        *op_errstr = gf_strdup (msg);
+                        goto out;
+                }
+
+                if (!peerinfo->connected) {
+                        snprintf (msg, sizeof (msg), "%s, is not connected at "
+                                  "the moment", hostname);
+                        *op_errstr = gf_strdup (msg);
+                        ret = -1;
+                        goto out;
+                }
+
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr)
+{
+        int                    ret            = -1;
+        uint32_t               cmd            = 0;
+        char                   msg[2048]      = {0,};
+        char                  *volname        = NULL;
+        char                  *brick          = NULL;
+        xlator_t              *this           = NULL;
+        glusterd_conf_t       *priv           = NULL;
+        glusterd_brickinfo_t  *brickinfo      = NULL;
+        glusterd_volinfo_t    *volinfo        = NULL;
+        dict_t                *vol_opts       = NULL;
+        gf_boolean_t           nfs_disabled   = _gf_false;
+        gf_boolean_t           shd_enabled    = _gf_true;
+
+        GF_ASSERT (dict);
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT(priv);
+
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        if (ret)
+                goto out;
+
+        if (cmd & GF_CLI_STATUS_ALL)
+                goto out;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Volume %s does not exist",
+                          volname);
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_is_volume_started (volinfo);
+        if (!ret) {
+                snprintf (msg, sizeof (msg), "Volume %s is not started",
+                          volname);
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                ret = -1;
+                goto out;
+        }
+
+        vol_opts = volinfo->dict;
+
+        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+                nfs_disabled = dict_get_str_boolean (vol_opts, "nfs.disable",
+                                                     _gf_false);
+                if (nfs_disabled) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg),
+                                  "NFS server is disabled for volume %s",
+                                  volname);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                        goto out;
+                }
+        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+                if (!glusterd_is_volume_replicate (volinfo)) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg),
+                                  "Volume %s is not of type replicate",
+                                  volname);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                        goto out;
+                }
+
+                shd_enabled = dict_get_str_boolean (vol_opts,
+                                                    "cluster.self-heal-daemon",
+                                                    _gf_true);
+                if (!shd_enabled) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg),
+                                  "Self-heal Daemon is disabled for volume %s",
+                                  volname);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                        goto out;
+                }
+
+        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+                ret = dict_get_str (dict, "brick", &brick);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                              &brickinfo);
+                if (ret) {
+                        snprintf (msg, sizeof(msg), "No brick %s in"
+                                  " volume %s", brick, volname);
+                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+
+ out:
+        if (ret) {
+                if (msg[0] != '\0')
+                        *op_errstr = gf_strdup (msg);
+                else
+                        *op_errstr = gf_strdup ("Validation Failed for Status");
+        }
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning: %d", ret);
+        return ret;
+}
+
+
+static gf_boolean_t
+glusterd_is_profile_on (glusterd_volinfo_t *volinfo)
+{
+        int                                     ret = -1;
+        gf_boolean_t                            is_latency_on = _gf_false;
+        gf_boolean_t                            is_fd_stats_on = _gf_false;
+
+        GF_ASSERT (volinfo);
+
+        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_DIAG_CNT_FOP_HITS);
+        if (ret != -1)
+                is_fd_stats_on = ret;
+        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_DIAG_LAT_MEASUREMENT);
+        if (ret != -1)
+                is_latency_on = ret;
+        if ((_gf_true == is_latency_on) &&
+            (_gf_true == is_fd_stats_on))
+                return _gf_true;
+        return _gf_false;
+}
+
+static int
+glusterd_op_stage_stats_volume (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        gf_boolean_t                            exists = _gf_false;
+        char                                    msg[2048] = {0,};
+        int32_t                                 stats_op = GF_CLI_STATS_NONE;
+        glusterd_volinfo_t                      *volinfo = NULL;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume name get failed");
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if ((!exists) || (ret < 0)) {
+                snprintf (msg, sizeof (msg), "Volume %s, "
+                         "doesn't exist", volname);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, "op", &stats_op);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume profile op get failed");
+                goto out;
+        }
+
+        if (GF_CLI_STATS_START == stats_op) {
+                if (_gf_true == glusterd_is_profile_on (volinfo)) {
+                        snprintf (msg, sizeof (msg), "Profile on Volume %s is"
+                                  " already started", volinfo->volname);
+                        ret = -1;
+                        goto out;
+                }
+
+        }
+        if ((GF_CLI_STATS_STOP == stats_op) ||
+            (GF_CLI_STATS_INFO == stats_op)) {
+                if (_gf_false == glusterd_is_profile_on (volinfo)) {
+                        snprintf (msg, sizeof (msg), "Profile on Volume %s is"
+                                  " not started", volinfo->volname);
+                        ret = -1;
+
+                        goto out;
+                }
+        }
+        if ((GF_CLI_STATS_TOP == stats_op) ||
+            (GF_CLI_STATS_INFO == stats_op)) {
+                if (_gf_false == glusterd_is_volume_started (volinfo)) {
+                        snprintf (msg, sizeof (msg), "Volume %s is not started.",
+                                  volinfo->volname);
+                        gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+        ret = 0;
+out:
+        if (msg[0] != '\0') {
+                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+static int
+_delete_reconfig_opt (dict_t *this, char *key, data_t *value, void *data)
+{
+        int32_t        *is_force = 0;
+
+        GF_ASSERT (data);
+        is_force = (int32_t*)data;
+
+        if (*is_force != 1) {
+                if (_gf_true == glusterd_check_voloption_flags (key,
+                                                         OPT_FLAG_FORCE)) {
+                /* indicate to caller that we don't set the option
+                 * due to being protected
+                 */
+                        *is_force = *is_force | GD_OP_PROTECTED;
+                        goto out;
+                } else {
+                        *is_force = *is_force | GD_OP_UNPROTECTED;
+                }
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "deleting dict with key=%s,value=%s",
+                key, value->data);
+        dict_del (this, key);
+out:
+        return 0;
+}
+
+static int
+_delete_reconfig_global_opt (dict_t *this, char *key, data_t *value, void *data)
+{
+        int32_t        *is_force = 0;
+
+        GF_ASSERT (data);
+        is_force = (int32_t*)data;
+
+        if (strcmp (GLUSTERD_GLOBAL_OPT_VERSION, key) == 0)
+                goto out;
+
+        _delete_reconfig_opt (this, key, value, data);
+out:
+        return 0;
+}
+
+static int
+glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key,
+                        int32_t *is_force)
+{
+        int                      ret = 0;
+        data_t                  *value = NULL;
+        char                    *key_fixed = NULL;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (volinfo->dict);
+        GF_ASSERT (key);
+
+        if (!strncmp(key, "all", 3))
+                dict_foreach (volinfo->dict, _delete_reconfig_opt, is_force);
+        else {
+                value = dict_get (volinfo->dict, key);
+                if (!value) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "no value set for option %s", key);
+                        goto out;
+                }
+                _delete_reconfig_opt (volinfo->dict, key, value, is_force);
+        }
+
+        gd_update_volume_op_versions (volinfo);
+
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to create volfile for"
+                        " 'volume reset'");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+                goto out;
+
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                ret = glusterd_nodesvcs_handle_reconfigure (volinfo);
+                if (ret)
+                        goto out;
+        }
+
+        ret = 0;
+
+out:
+        GF_FREE (key_fixed);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
+{
+        char            *key            = NULL;
+        char            *key_fixed      = NULL;
+        int             ret             = -1;
+        int32_t         is_force        = 0;
+        glusterd_conf_t *conf           = NULL;
+        dict_t          *dup_opt        = NULL;
+        gf_boolean_t    all             = _gf_false;
+        char            *next_version   = NULL;
+        gf_boolean_t    quorum_action   = _gf_false;
+
+        conf = this->private;
+        ret = dict_get_str (dict, "key", &key);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get key");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "force", &is_force);
+        if (ret)
+                is_force = 0;
+
+        if (strcmp (key, "all")) {
+                ret = glusterd_check_option_exists (key, &key_fixed);
+                if (ret <= 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Option %s does not "
+                                "exist", key);
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                all = _gf_true;
+        }
+
+        if (key_fixed)
+                key = key_fixed;
+
+        ret = -1;
+        dup_opt = dict_new ();
+        if (!dup_opt)
+                goto out;
+        if (!all) {
+                dict_copy (conf->opts, dup_opt);
+                dict_del (dup_opt, key);
+        }
+        ret = glusterd_get_next_global_opt_version_str (conf->opts,
+                                                        &next_version);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_options (this, dup_opt);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_quorum_changed (conf->opts, key, NULL))
+                quorum_action = _gf_true;
+
+        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                               next_version);
+        if (ret)
+                goto out;
+        else
+                next_version = NULL;
+
+        if (!all) {
+                dict_del (conf->opts, key);
+        } else {
+                dict_foreach (conf->opts, _delete_reconfig_global_opt,
+                              &is_force);
+        }
+out:
+        GF_FREE (key_fixed);
+        if (dup_opt)
+                dict_unref (dup_opt);
+
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        GF_FREE (next_version);
+        return ret;
+}
+
+static int
+glusterd_op_reset_volume (dict_t *dict, char **op_rspstr)
+{
+        glusterd_volinfo_t      *volinfo    = NULL;
+        int                     ret         = -1;
+        char                    *volname    = NULL;
+        char                    *key        = NULL;
+        char                    *key_fixed  = NULL;
+        int32_t                 is_force    = 0;
+        gf_boolean_t            quorum_action = _gf_false;
+        xlator_t                *this         = NULL;
+
+        this = THIS;
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name" );
+                goto out;
+        }
+
+        if (strcasecmp (volname, "all") == 0) {
+                ret = glusterd_op_reset_all_volume_options (this, dict);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "force", &is_force);
+        if (ret)
+                is_force = 0;
+
+        ret = dict_get_str (dict, "key", &key);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get option key");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        if (strcmp (key, "all") &&
+            glusterd_check_option_exists (key, &key_fixed) != 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "volinfo dict inconsistency: option %s not found",
+                        key);
+                ret = -1;
+                goto out;
+        }
+        if (key_fixed)
+                key = key_fixed;
+
+        if (glusterd_is_quorum_changed (volinfo->dict, key, NULL))
+                quorum_action = _gf_true;
+
+        ret = glusterd_options_reset (volinfo, key, &is_force);
+        if (ret == -1) {
+                gf_asprintf(op_rspstr, "Volume reset : failed");
+        } else if (is_force & GD_OP_PROTECTED) {
+                if (is_force & GD_OP_UNPROTECTED) {
+                        gf_asprintf (op_rspstr, "All unprotected fields were"
+                                     " reset. To reset the protected fields,"
+                                     " use 'force'.");
+                } else {
+                        ret = -1;
+                        gf_asprintf (op_rspstr, "'%s' is protected. To reset"
+                                     " use 'force'.", key);
+                }
+        }
+
+out:
+        GF_FREE (key_fixed);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+
+        gf_log (this->name, GF_LOG_DEBUG, "'volume reset' returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_stop_bricks (glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t                    *brickinfo = NULL;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+            /*TODO: Need to change @del_brick in brick_stop to _gf_true
+             * once we enable synctask in peer rpc prog */
+                if (glusterd_brick_stop (volinfo, brickinfo, _gf_false))
+                        return -1;
+        }
+
+        return 0;
+}
+
+int
+glusterd_start_bricks (glusterd_volinfo_t *volinfo)
+{
+        int                      ret            = -1;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_brick_start (volinfo, brickinfo, _gf_false);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Failed to start %s:%s for %s",
+                                brickinfo->hostname, brickinfo->path,
+                                volinfo->volname);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+glusterd_op_set_all_volume_options (xlator_t *this, dict_t *dict)
+{
+        char            *key            = NULL;
+        char            *key_fixed      = NULL;
+        char            *value          = NULL;
+        char            *dup_value      = NULL;
+        int             ret             = -1;
+        glusterd_conf_t *conf           = NULL;
+        dict_t          *dup_opt        = NULL;
+        char            *next_version   = NULL;
+        gf_boolean_t    quorum_action   = _gf_false;
+
+        conf = this->private;
+        ret = dict_get_str (dict, "key1", &key);
+        if (ret)
+                goto out;
+
+        ret = dict_get_str (dict, "value1", &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "invalid key,value pair in 'volume set'");
+                goto out;
+        }
+        ret = glusterd_check_option_exists (key, &key_fixed);
+        if (ret <= 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid key %s", key);
+                ret = -1;
+                goto out;
+        }
+
+        if (key_fixed)
+                key = key_fixed;
+
+        ret = -1;
+        dup_opt = dict_new ();
+        if (!dup_opt)
+                goto out;
+        dict_copy (conf->opts, dup_opt);
+        ret = dict_set_str (dup_opt, key, value);
+        if (ret)
+                goto out;
+
+        ret = glusterd_get_next_global_opt_version_str (conf->opts,
+                                                        &next_version);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+        if (ret)
+                goto out;
+
+        dup_value = gf_strdup (value);
+        if (!dup_value)
+                goto out;
+
+        ret = glusterd_store_options (this, dup_opt);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_quorum_changed (conf->opts, key, value))
+                quorum_action = _gf_true;
+
+        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                               next_version);
+        if (ret)
+                goto out;
+        else
+                next_version = NULL;
+
+        ret = dict_set_dynstr (conf->opts, key, dup_value);
+        if (ret)
+                goto out;
+out:
+        GF_FREE (key_fixed);
+        if (dup_opt)
+                dict_unref (dup_opt);
+
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        GF_FREE (next_version);
+        return ret;
+}
+
+static int
+glusterd_op_set_volume (dict_t *dict)
+{
+        int                                      ret = 0;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        char                                    *volname = NULL;
+        xlator_t                                *this = NULL;
+        glusterd_conf_t                         *priv = NULL;
+        int                                      count = 1;
+        char                                    *key = NULL;
+        char                                    *key_fixed = NULL;
+        char                                    *value = NULL;
+        char                                     str[50] = {0, };
+        char                                    *op_errstr = NULL;
+        gf_boolean_t                             global_opt    = _gf_false;
+        gf_boolean_t                             global_opts_set = _gf_false;
+        glusterd_volinfo_t                      *voliter = NULL;
+        int32_t                                  dict_count = 0;
+        gf_boolean_t                             check_op_version = _gf_false;
+        uint32_t                                 new_op_version = 0;
+        gf_boolean_t                            quorum_action  = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_int32 (dict, "count", &dict_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Count(dict),not set in Volume-Set");
+                goto out;
+        }
+
+        if (dict_count == 0) {
+                ret = glusterd_volset_help (NULL, &op_errstr);
+                if (ret) {
+                        op_errstr = (op_errstr)? op_errstr:
+                                     "Volume set help internal error";
+                        gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+                }
+                goto out;
+         }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        if (strcasecmp (volname, "all") == 0) {
+                ret = glusterd_op_set_all_volume_options (this, dict);
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        // TODO: Remove this once v3.3 compatability is not required
+        check_op_version = dict_get_str_boolean (dict, "check-op-version",
+                                                 _gf_false);
+
+        if (check_op_version) {
+                ret = dict_get_uint32 (dict, "new-op-version", &new_op_version);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get new op-version from dict");
+                        goto out;
+                }
+        }
+
+        for (count = 1; ret != -1 ; count++) {
+
+                sprintf (str, "key%d", count);
+                ret = dict_get_str (dict, str, &key);
+                if (ret)
+                        break;
+
+                sprintf (str, "value%d", count);
+                ret = dict_get_str (dict, str, &value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid key,value pair in 'volume set'");
+                        ret = -1;
+                        goto out;
+                }
+
+                if (strcmp (key, "config.memory-accounting") == 0) {
+                        ret = gf_string2boolean (value,
+                                                 &volinfo->memory_accounting);
+                }
+
+                if (strcmp (key, "config.transport") == 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "changing transport-type for volume %s to %s",
+                                volname, value);
+                        ret = 0;
+                        if (strcasecmp (value, "rdma") == 0) {
+                                volinfo->transport_type = GF_TRANSPORT_RDMA;
+                        } else if (strcasecmp (value, "tcp") == 0) {
+                                volinfo->transport_type = GF_TRANSPORT_TCP;
+                        } else if ((strcasecmp (value, "tcp,rdma") == 0) ||
+                                   (strcasecmp (value, "rdma,tcp") == 0)) {
+                                volinfo->transport_type =
+                                        GF_TRANSPORT_BOTH_TCP_RDMA;
+                        } else {
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                if (!is_key_glusterd_hooks_friendly (key)) {
+                        ret = glusterd_check_option_exists (key, &key_fixed);
+                        GF_ASSERT (ret);
+                        if (ret <= 0) {
+                                key_fixed = NULL;
+                                goto out;
+                        }
+                }
+
+                global_opt = _gf_false;
+                if (glusterd_check_globaloption (key)) {
+                        global_opt = _gf_true;
+                        global_opts_set = _gf_true;
+                }
+
+                if (!global_opt)
+                        value = gf_strdup (value);
+
+                if (!value) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set the options in 'volume set'");
+                        ret = -1;
+                        goto out;
+                }
+
+                if (key_fixed)
+                        key = key_fixed;
+
+                if (glusterd_is_quorum_changed (volinfo->dict, key, value))
+                        quorum_action = _gf_true;
+
+                if (global_opt) {
+                       list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                               value = gf_strdup (value);
+                               ret = dict_set_dynstr (voliter->dict, key, value);
+                               if (ret)
+                                       goto out;
+                       }
+                } else {
+                        ret = dict_set_dynstr (volinfo->dict, key, value);
+                        if (ret)
+                                goto out;
+                }
+
+                if (key_fixed) {
+                        GF_FREE (key_fixed);
+                        key_fixed = NULL;
+                }
+        }
+
+        if (count == 1) {
+                gf_log (this->name, GF_LOG_ERROR, "No options received ");
+                ret = -1;
+                goto out;
+        }
+
+        /* Update the cluster op-version before regenerating volfiles so that
+         * correct volfiles are generated
+         */
+        if (new_op_version > priv->op_version) {
+                priv->op_version = new_op_version;
+                ret = glusterd_store_global_info (this);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store op-version");
+                        goto out;
+                }
+        }
+
+        if (!global_opts_set) {
+                gd_update_volume_op_versions (volinfo);
+                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to create volfile for"
+                                " 'volume set'");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret)
+                        goto out;
+
+                if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                        ret = glusterd_nodesvcs_handle_reconfigure (volinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                         "Unable to restart NFS-Server");
+                                goto out;
+                        }
+                }
+
+        } else {
+                list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                        volinfo = voliter;
+                        gd_update_volume_op_versions (volinfo);
+                        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to create volfile for"
+                                        " 'volume set'");
+                                ret = -1;
+                                goto out;
+                        }
+
+                        ret = glusterd_store_volinfo (volinfo,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                        if (ret)
+                                goto out;
+
+                        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                                ret = glusterd_nodesvcs_handle_reconfigure (volinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Unable to restart NFS-Server");
+                                        goto out;
+                                }
+                        }
+                }
+        }
+
+ out:
+        GF_FREE (key_fixed);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        return ret;
+}
+
+
+static int
+glusterd_op_sync_volume (dict_t *dict, char **op_errstr,
+                         dict_t *rsp_dict)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        char                                    *hostname = NULL;
+        char                                    msg[2048] = {0,};
+        int                                     count = 1;
+        int                                     vol_count = 0;
+        glusterd_conf_t                         *priv = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "hostname couldn't be "
+                          "retrieved from msg");
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        if (!gf_is_local_addr (hostname)) {
+                ret = 0;
+                goto out;
+        }
+
+        //volname is not present in case of sync all
+        ret = dict_get_str (dict, "volname", &volname);
+        if (!ret) {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Volume with name: %s "
+                                "not exists", volname);
+                        goto out;
+                }
+        }
+
+        if (!rsp_dict) {
+                //this should happen only on source
+                ret = 0;
+                goto out;
+        }
+
+        if (volname) {
+                ret = glusterd_add_volume_to_dict (volinfo, rsp_dict,
+                                                   1);
+                vol_count = 1;
+        } else {
+                list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                        ret = glusterd_add_volume_to_dict (volinfo,
+                                                           rsp_dict, count);
+                        if (ret)
+                                goto out;
+
+                        vol_count = count++;
+                }
+        }
+        ret = dict_set_int32 (rsp_dict, "count", vol_count);
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_add_profile_volume_options (glusterd_volinfo_t *volinfo)
+{
+        int                                     ret = -1;
+        char                                    *latency_key = NULL;
+        char                                    *fd_stats_key = NULL;
+
+        GF_ASSERT (volinfo);
+
+        latency_key = VKEY_DIAG_LAT_MEASUREMENT;
+        fd_stats_key = VKEY_DIAG_CNT_FOP_HITS;
+
+        ret = dict_set_str (volinfo->dict, latency_key, "on");
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "failed to set the volume %s "
+                        "option %s value %s",
+                        volinfo->volname, latency_key, "on");
+                goto out;
+        }
+
+        ret = dict_set_str (volinfo->dict, fd_stats_key, "on");
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "failed to set the volume %s "
+                        "option %s value %s",
+                        volinfo->volname, fd_stats_key, "on");
+                goto out;
+        }
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static void
+glusterd_remove_profile_volume_options (glusterd_volinfo_t *volinfo)
+{
+        char                                    *latency_key = NULL;
+        char                                    *fd_stats_key = NULL;
+
+        GF_ASSERT (volinfo);
+
+        latency_key = VKEY_DIAG_LAT_MEASUREMENT;
+        fd_stats_key = VKEY_DIAG_CNT_FOP_HITS;
+        dict_del (volinfo->dict, latency_key);
+        dict_del (volinfo->dict, fd_stats_key);
+}
+
+static int
+glusterd_op_stats_volume (dict_t *dict, char **op_errstr,
+                          dict_t *rsp_dict)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        char                                    msg[2048] = {0,};
+        glusterd_volinfo_t                      *volinfo = NULL;
+        int32_t                                 stats_op = GF_CLI_STATS_NONE;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume %s does not exists",
+                          volname);
+
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "op", &stats_op);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "volume profile op get failed");
+                goto out;
+        }
+
+        switch (stats_op) {
+        case GF_CLI_STATS_START:
+                ret = glusterd_add_profile_volume_options (volinfo);
+                if (ret)
+                        goto out;
+                break;
+        case GF_CLI_STATS_STOP:
+                glusterd_remove_profile_volume_options (volinfo);
+                break;
+        case GF_CLI_STATS_INFO:
+        case GF_CLI_STATS_TOP:
+                //info is already collected in brick op.
+                //just goto out;
+                ret = 0;
+                goto out;
+                break;
+        default:
+                GF_ASSERT (0);
+                gf_log ("glusterd", GF_LOG_ERROR, "Invalid profile op: %d",
+                        stats_op);
+                ret = -1;
+                goto out;
+                break;
+        }
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to create volfile for"
+                                          " 'volume set'");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (volinfo,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+                goto out;
+
+        if (GLUSTERD_STATUS_STARTED == volinfo->status)
+                ret = glusterd_nodesvcs_handle_reconfigure (volinfo);
+
+        ret = 0;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+_add_brick_name_to_dict (dict_t *dict, char *key, glusterd_brickinfo_t *brick)
+{
+        int     ret = -1;
+        char    tmp[1024] = {0,};
+        char    *brickname = NULL;
+        xlator_t *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key);
+        GF_ASSERT (brick);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        snprintf (tmp, sizeof (tmp), "%s:%s", brick->hostname, brick->path);
+        brickname = gf_strdup (tmp);
+        if (!brickname) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to dup brick name");
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dict, key, brickname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to add brick name to dict");
+                goto out;
+        }
+        brickname = NULL;
+out:
+        if (brickname)
+                GF_FREE (brickname);
+        return ret;
+}
+
+static int
+_add_remove_bricks_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
+                            char *prefix)
+{
+        int             ret = -1;
+        int             count = 0;
+        int             i = 0;
+        char            brick_key[1024] = {0,};
+        char            dict_key[1024] ={0,};
+        char            *brick = NULL;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+        GF_ASSERT (prefix);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_int32 (volinfo->rebal.dict, "count", &count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get brick count");
+                goto out;
+        }
+
+        snprintf (dict_key, sizeof (dict_key), "%s.count", prefix);
+        ret = dict_set_int32 (dict, dict_key, count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set brick count in dict");
+                goto out;
+        }
+
+        for (i = 1; i <= count; i++) {
+                memset (brick_key, 0, sizeof (brick_key));
+                snprintf (brick_key, sizeof (brick_key), "brick%d", i);
+
+                ret = dict_get_str (volinfo->rebal.dict, brick_key, &brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get %s", brick_key);
+                        goto out;
+                }
+
+                memset (dict_key, 0, sizeof (dict_key));
+                snprintf (dict_key, sizeof (dict_key), "%s.%s", prefix,
+                          brick_key);
+                ret = dict_set_str (dict, dict_key, brick);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add brick to dict");
+                        goto out;
+                }
+                brick = NULL;
+        }
+
+out:
+        return ret;
+}
+
+/* This adds the respective task-id and all available parameters of a task into
+ * a dictionary
+ */
+static int
+_add_task_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
+{
+
+        int             ret = -1;
+        char            key[128] = {0,};
+        char            *uuid_str = NULL;
+        int             status = 0;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        switch (op) {
+        case GD_OP_REMOVE_BRICK:
+                snprintf (key, sizeof (key), "task%d", index);
+                ret = _add_remove_bricks_to_dict (dict, volinfo, key);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add remove bricks to dict");
+                        goto out;
+                }
+        case GD_OP_REBALANCE:
+                uuid_str = gf_strdup (uuid_utoa (volinfo->rebal.rebalance_id));
+                status = volinfo->rebal.defrag_status;
+                break;
+
+        case GD_OP_REPLACE_BRICK:
+                snprintf (key, sizeof (key), "task%d.src-brick", index);
+                ret = _add_brick_name_to_dict (dict, key,
+                                               volinfo->rep_brick.src_brick);
+                if (ret)
+                        goto out;
+                memset (key, 0, sizeof (key));
+
+                snprintf (key, sizeof (key), "task%d.dst-brick", index);
+                ret = _add_brick_name_to_dict (dict, key,
+                                               volinfo->rep_brick.dst_brick);
+                if (ret)
+                        goto out;
+                memset (key, 0, sizeof (key));
+
+                uuid_str = gf_strdup (uuid_utoa (volinfo->rep_brick.rb_id));
+                status = volinfo->rep_brick.rb_status;
+                break;
+
+        default:
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "%s operation doesn't have a"
+                        " task_id", gd_op_list[op]);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "task%d.type", index);
+        ret = dict_set_str (dict, key, (char *)gd_op_list[op]);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting task type in dict");
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "task%d.id", index);
+
+        if (!uuid_str)
+                goto out;
+        ret = dict_set_dynstr (dict, key, uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting task id in dict");
+                goto out;
+        }
+        uuid_str = NULL;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "task%d.status", index);
+        ret = dict_set_int32 (dict, key, status);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting task status in dict");
+                goto out;
+        }
+
+out:
+        if (uuid_str)
+                GF_FREE (uuid_str);
+        return ret;
+}
+
+static int
+glusterd_aggregate_task_status (dict_t *rsp_dict, glusterd_volinfo_t *volinfo)
+{
+        int        ret   = -1;
+        int        tasks = 0;
+        xlator_t  *this  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                ret = _add_task_to_dict (rsp_dict, volinfo, volinfo->rebal.op,
+                                         tasks);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add task details to dict");
+                        goto out;
+                }
+                tasks++;
+        }
+
+        if (!uuid_is_null (volinfo->rep_brick.rb_id)) {
+                ret = _add_task_to_dict (rsp_dict, volinfo, GD_OP_REPLACE_BRICK,
+                                         tasks);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add task details to dict");
+                        goto out;
+                }
+                tasks++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "tasks", tasks);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting tasks count in dict");
+                goto out;
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
+glusterd_op_status_volume (dict_t *dict, char **op_errstr,
+                           dict_t *rsp_dict)
+{
+        int                     ret             = -1;
+        int                     node_count      = 0;
+        int                     brick_index     = -1;
+        int                     other_count     = 0;
+        int                     other_index     = 0;
+        uint32_t                cmd             = 0;
+        char                   *volname         = NULL;
+        char                   *brick           = NULL;
+        xlator_t               *this            = NULL;
+        glusterd_volinfo_t     *volinfo         = NULL;
+        glusterd_brickinfo_t   *brickinfo       = NULL;
+        glusterd_conf_t        *priv            = NULL;
+        dict_t                 *vol_opts        = NULL;
+        gf_boolean_t            nfs_disabled    = _gf_false;
+        gf_boolean_t            shd_enabled     = _gf_true;
+        gf_boolean_t            origin_glusterd = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        GF_ASSERT (dict);
+
+        origin_glusterd = is_origin_glusterd (dict);
+
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        if (ret)
+                goto out;
+
+        if (origin_glusterd) {
+                ret = 0;
+                if ((cmd & GF_CLI_STATUS_ALL)) {
+                        ret = glusterd_get_all_volnames (rsp_dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to get all volume "
+                                        "names for status");
+                }
+        }
+
+        ret = dict_set_uint32 (rsp_dict, "cmd", cmd);
+        if (ret)
+                goto out;
+
+        if (cmd & GF_CLI_STATUS_ALL)
+                goto out;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret)
+                goto out;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Volume with name: %s "
+                        "does not exist", volname);
+                goto out;
+        }
+        vol_opts = volinfo->dict;
+
+        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+                ret = glusterd_add_node_to_dict ("nfs", rsp_dict, 0, vol_opts);
+                if (ret)
+                        goto out;
+                other_count++;
+                node_count++;
+
+        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+                ret = glusterd_add_node_to_dict ("glustershd", rsp_dict, 0,
+                                                 vol_opts);
+                if (ret)
+                        goto out;
+                other_count++;
+                node_count++;
+
+        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+                ret = dict_get_str (dict, "brick", &brick);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_volume_brickinfo_get_by_brick (brick,
+                                                              volinfo,
+                                                              &brickinfo);
+                if (ret)
+                        goto out;
+
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        goto out;
+
+                glusterd_add_brick_to_dict (volinfo, brickinfo, rsp_dict,
+                                            ++brick_index);
+                if (cmd & GF_CLI_STATUS_DETAIL)
+                        glusterd_add_brick_detail_to_dict (volinfo, brickinfo,
+                                                           rsp_dict,
+                                                           brick_index);
+                node_count++;
+
+        } else if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
+                ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
+                goto out;
+
+        } else {
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        brick_index++;
+                        if (uuid_compare (brickinfo->uuid, MY_UUID))
+                                continue;
+
+                        glusterd_add_brick_to_dict (volinfo, brickinfo,
+                                                    rsp_dict, brick_index);
+
+                        if (cmd & GF_CLI_STATUS_DETAIL) {
+                                glusterd_add_brick_detail_to_dict (volinfo,
+                                                                   brickinfo,
+                                                                   rsp_dict,
+                                                                   brick_index);
+                        }
+                        node_count++;
+                }
+
+                if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
+                        other_index = brick_index + 1;
+
+                        nfs_disabled = dict_get_str_boolean (vol_opts,
+                                                             "nfs.disable",
+                                                             _gf_false);
+                        if (!nfs_disabled) {
+                                ret = glusterd_add_node_to_dict ("nfs",
+                                                                 rsp_dict,
+                                                                 other_index,
+                                                                 vol_opts);
+                                if (ret)
+                                        goto out;
+                                other_index++;
+                                other_count++;
+                                node_count++;
+                        }
+
+                        shd_enabled = dict_get_str_boolean
+                                        (vol_opts, "cluster.self-heal-daemon",
+                                         _gf_true);
+                        if (glusterd_is_volume_replicate (volinfo)
+                            && shd_enabled) {
+                                ret = glusterd_add_node_to_dict ("glustershd",
+                                                                 rsp_dict,
+                                                                 other_index,
+                                                                 vol_opts);
+                                if (ret)
+                                        goto out;
+                                other_count++;
+                                node_count++;
+                        }
+                }
+        }
+
+        ret = dict_set_int32 (rsp_dict, "brick-index-max", brick_index);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting brick-index-max to dict");
+                goto out;
+        }
+        ret = dict_set_int32 (rsp_dict, "other-count", other_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting other-count to dict");
+                goto out;
+        }
+        ret = dict_set_int32 (rsp_dict, "count", node_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Error setting node count to dict");
+                goto out;
+        }
+
+        /* Active tasks */
+        /* Tasks are added only for normal volume status request for either a
+         * single volume or all volumes, and only by the origin glusterd
+         */
+        if (((cmd & GF_CLI_STATUS_MASK) != GF_CLI_STATUS_NONE) ||
+            !(cmd & (GF_CLI_STATUS_VOL | GF_CLI_STATUS_ALL)) ||
+            !origin_glusterd)
+                goto out;
+
+        ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
+        if (ret)
+                goto out;
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_none (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int ret = 0;
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_send_lock (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                   ret      = 0;
+        rpc_clnt_procedure_t *proc     = NULL;
+        glusterd_conf_t      *priv     = NULL;
+        xlator_t             *this     = NULL;
+        glusterd_peerinfo_t  *peerinfo = NULL;
+        uint32_t             pending_count = 0;
+        dict_t               *dict     = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                GF_ASSERT (peerinfo);
+
+                if (!peerinfo->connected || !peerinfo->mgmt)
+                        continue;
+                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+                        continue;
+
+                /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+                if (priv->op_version < 3) {
+                        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_LOCK];
+                        if (proc->fn) {
+                                ret = proc->fn (NULL, this, peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send lock request "
+                                                "for operation 'Volume %s' to "
+                                                "peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                } else {
+                        dict = glusterd_op_get_ctx ();
+                        dict_ref (dict);
+
+                        proc = &peerinfo->mgmt_v3->proctable
+                                                   [GLUSTERD_MGMT_V3_LOCK];
+                        if (proc->fn) {
+                                ret = dict_set_static_ptr (dict, "peerinfo",
+                                                           peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "failed to set peerinfo");
+                                        dict_unref (dict);
+                                        goto out;
+                                }
+
+                                ret = proc->fn (NULL, this, dict);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send mgmt_v3 lock "
+                                                "request for operation "
+                                                "'Volume %s' to peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        dict_unref (dict);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                }
+        }
+
+        opinfo.pending_count = pending_count;
+        if (!opinfo.pending_count)
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+static int
+glusterd_op_ac_send_unlock (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                   ret      = 0;
+        rpc_clnt_procedure_t *proc     = NULL;
+        glusterd_conf_t      *priv     = NULL;
+        xlator_t             *this     = NULL;
+        glusterd_peerinfo_t  *peerinfo = NULL;
+        uint32_t             pending_count = 0;
+        dict_t               *dict     = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                GF_ASSERT (peerinfo);
+
+                if (!peerinfo->connected || !peerinfo->mgmt)
+                        continue;
+                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+                        continue;
+
+                /* Based on the op_version, release the *
+                 * cluster or mgmt_v3 lock */
+                if (priv->op_version < 3) {
+                        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_UNLOCK];
+                        if (proc->fn) {
+                                ret = proc->fn (NULL, this, peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send unlock request "
+                                                "for operation 'Volume %s' to "
+                                                "peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                } else {
+                        dict = glusterd_op_get_ctx ();
+                        dict_ref (dict);
+
+                        proc = &peerinfo->mgmt_v3->proctable
+                                                   [GLUSTERD_MGMT_V3_UNLOCK];
+                        if (proc->fn) {
+                                ret = dict_set_static_ptr (dict, "peerinfo",
+                                                           peerinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "failed to set peerinfo");
+                                        dict_unref (dict);
+                                        goto out;
+                                }
+
+                                ret = proc->fn (NULL, this, dict);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "Failed to send mgmt_v3 unlock "
+                                                "request for operation "
+                                                "'Volume %s' to peer %s",
+                                                gd_op_list[opinfo.op],
+                                                peerinfo->hostname);
+                                        dict_unref (dict);
+                                        continue;
+                                }
+                                pending_count++;
+                        }
+                }
+        }
+
+        opinfo.pending_count = pending_count;
+        if (!opinfo.pending_count)
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+static int
+glusterd_op_ac_ack_drain (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int ret = 0;
+
+        if (opinfo.pending_count > 0)
+                opinfo.pending_count--;
+
+        if (!opinfo.pending_count)
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                                   &event->txn_id, NULL);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_send_unlock_drain (glusterd_op_sm_event_t *event, void *ctx)
+{
+        return glusterd_op_ac_ack_drain (event, ctx);
+}
+
+static int
+glusterd_op_ac_lock (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int32_t                         ret             = 0;
+        char                           *volname         = NULL;
+        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
+        glusterd_conf_t                *priv            = NULL;
+        xlator_t                       *this            = NULL;
+
+        GF_ASSERT (event);
+        GF_ASSERT (ctx);
+
+        this = THIS;
+        priv = this->private;
+
+        lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
+
+        /* If the req came from a node running on older op_version
+         * the dict won't be present. Based on it acquiring a cluster
+         * or mgmt_v3 lock */
+        if (lock_ctx->dict == NULL) {
+                ret = glusterd_lock (lock_ctx->uuid);
+                glusterd_op_lock_send_resp (lock_ctx->req, ret);
+        } else {
+                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+                else {
+                        ret = glusterd_mgmt_v3_lock (volname, lock_ctx->uuid,
+                                                     "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to acquire lock for %s",
+                                        volname);
+                }
+
+                glusterd_op_mgmt_v3_lock_send_resp (lock_ctx->req,
+                                                    &event->txn_id, ret);
+
+                dict_unref (lock_ctx->dict);
+        }
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Lock Returned %d", ret);
+        return ret;
+}
+
+static int
+glusterd_op_ac_unlock (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int32_t                         ret             = 0;
+        char                           *volname         = NULL;
+        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
+        glusterd_conf_t                *priv            = NULL;
+        xlator_t                       *this            = NULL;
+
+
+        GF_ASSERT (event);
+        GF_ASSERT (ctx);
+
+        this = THIS;
+        priv = this->private;
+
+        lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
+
+        /* If the req came from a node running on older op_version
+         * the dict won't be present. Based on it releasing the cluster
+         * or mgmt_v3 lock */
+        if (lock_ctx->dict == NULL) {
+                ret = glusterd_unlock (lock_ctx->uuid);
+                glusterd_op_unlock_send_resp (lock_ctx->req, ret);
+        } else {
+                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+                else {
+                        ret = glusterd_mgmt_v3_unlock (volname, lock_ctx->uuid,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s", volname);
+                }
+
+                glusterd_op_mgmt_v3_unlock_send_resp (lock_ctx->req,
+                                                      &event->txn_id, ret);
+
+                dict_unref (lock_ctx->dict);
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
+
+        if (priv->pending_quorum_action)
+                glusterd_do_quorum_action ();
+        return ret;
+}
+
+static int
+glusterd_op_ac_local_unlock (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int              ret          = 0;
+        uuid_t          *originator   = NULL;
+
+        GF_ASSERT (event);
+        GF_ASSERT (ctx);
+
+        originator = (uuid_t *) ctx;
+
+        ret = glusterd_unlock (*originator);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_rcvd_lock_acc (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+
+        GF_ASSERT (event);
+
+        if (opinfo.pending_count > 0)
+                opinfo.pending_count--;
+
+        if (opinfo.pending_count > 0)
+                goto out;
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
+                                           &event->txn_id, NULL);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+out:
+        return ret;
+}
+
+static int
+glusterd_dict_set_volid (dict_t *dict, char *volname, char **op_errstr)
+{
+        int                     ret = -1;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *volid = NULL;
+        char                    msg[1024] = {0,};
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dict || !volname)
+                goto out;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                goto out;
+        }
+        volid = gf_strdup (uuid_utoa (volinfo->volume_id));
+        if (!volid) {
+                ret = -1;
+                goto out;
+        }
+        ret = dict_set_dynstr (dict, "vol-id", volid);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to set volume id of volume"
+                          " %s", volname);
+                goto out;
+        }
+out:
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        return ret;
+}
+
+int
+glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
+{
+        int                     ret = -1;
+        void                    *ctx = NULL;
+        dict_t                  *dict = NULL;
+        dict_t                  *req_dict = NULL;
+        glusterd_op_t           op = GD_OP_NONE;
+        char                    *volname = NULL;
+        uint32_t                status_cmd = GF_CLI_STATUS_NONE;
+        char                    *errstr = NULL;
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        req_dict = dict_new ();
+        if (!req_dict)
+                goto out;
+
+        if (!op_ctx) {
+                op  = glusterd_op_get_op ();
+                ctx = (void*)glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log (this->name, GF_LOG_ERROR, "Null Context for "
+                                "op %d", op);
+                        ret = -1;
+                        goto out;
+                }
+
+        } else {
+#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
+                ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, (int32_t*)&op);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get volume"
+                                " operation");
+                        goto out;
+                }
+                ctx = op_ctx;
+#undef GD_SYNC_OPCODE_KEY
+        }
+
+        dict = ctx;
+        switch (op) {
+                case GD_OP_CREATE_VOLUME:
+                        {
+                                ++glusterfs_port;
+                                ret = dict_set_int32 (dict, "port",
+                                                      glusterfs_port);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to set port in "
+                                                "dictionary");
+                                        goto out;
+                                }
+                                dict_copy (dict, req_dict);
+                        }
+                        break;
+
+                case GD_OP_GSYNC_CREATE:
+                case GD_OP_GSYNC_SET:
+                        {
+                                ret = glusterd_op_gsync_args_get (dict,
+                                                                  &errstr,
+                                                                  &volname,
+                                                                  NULL, NULL);
+                                if (ret == 0) {
+                                        ret = glusterd_dict_set_volid
+                                                (dict, volname, op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
+                                dict_copy (dict, req_dict);
+                        }
+                        break;
+
+                case GD_OP_SET_VOLUME:
+                        {
+                                ret = dict_get_str (dict, "volname", &volname);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_CRITICAL,
+                                                "volname is not present in "
+                                                "operation ctx");
+                                        goto out;
+                                }
+                                if (strcmp (volname, "help") &&
+                                    strcmp (volname, "help-xml") &&
+                                    strcasecmp (volname, "all")) {
+                                        ret = glusterd_dict_set_volid
+                                                 (dict, volname, op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
+                                dict_destroy (req_dict);
+                                req_dict = dict_ref (dict);
+                        }
+                        break;
+
+                case GD_OP_SYNC_VOLUME:
+                        {
+                                dict_copy (dict, req_dict);
+                                break;
+                        }
+
+                case GD_OP_REMOVE_BRICK:
+                        {
+                                dict_t *dict = ctx;
+                                ret = dict_get_str (dict, "volname", &volname);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_CRITICAL,
+                                                "volname is not present in "
+                                                "operation ctx");
+                                        goto out;
+                                }
+
+                                ret = glusterd_dict_set_volid (dict, volname,
+                                                               op_errstr);
+                                if (ret)
+                                        goto out;
+
+                                dict_destroy (req_dict);
+                                req_dict = dict_ref (dict);
+                        }
+                        break;
+
+                case GD_OP_STATUS_VOLUME:
+                        {
+                                ret = dict_get_uint32 (dict, "cmd",
+                                                       &status_cmd);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Status command not present "
+                                                "in op ctx");
+                                        goto out;
+                                }
+                                if (GF_CLI_STATUS_ALL & status_cmd) {
+                                        dict_copy (dict, req_dict);
+                                        break;
+                                }
+                        }
+                        /*fall-through*/
+                case GD_OP_DELETE_VOLUME:
+                case GD_OP_START_VOLUME:
+                case GD_OP_STOP_VOLUME:
+                case GD_OP_ADD_BRICK:
+                case GD_OP_REPLACE_BRICK:
+                case GD_OP_RESET_VOLUME:
+                case GD_OP_LOG_ROTATE:
+                case GD_OP_QUOTA:
+                case GD_OP_PROFILE_VOLUME:
+                case GD_OP_REBALANCE:
+                case GD_OP_HEAL_VOLUME:
+                case GD_OP_STATEDUMP_VOLUME:
+                case GD_OP_CLEARLOCKS_VOLUME:
+                case GD_OP_DEFRAG_BRICK_VOLUME:
+                        {
+                                ret = dict_get_str (dict, "volname", &volname);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_CRITICAL,
+                                                "volname is not present in "
+                                                "operation ctx");
+                                        goto out;
+                                }
+
+                                if (strcasecmp (volname, "all")) {
+                                        ret = glusterd_dict_set_volid (dict,
+                                                                       volname,
+                                                                     op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
+                                dict_copy (dict, req_dict);
+                        }
+                        break;
+
+                case GD_OP_COPY_FILE:
+                        {
+                                dict_copy (dict, req_dict);
+                                break;
+                        }
+
+                case GD_OP_SYS_EXEC:
+                        {
+                                dict_copy (dict, req_dict);
+                                break;
+                        }
+
+                default:
+                        break;
+        }
+
+        *req = req_dict;
+        ret = 0;
+
+out:
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_get_op (xlator_t *this, glusterd_op_t op, dict_t *dict)
+{
+        char            *key = NULL;
+        char            *volname = NULL;
+        int             ret = 0;
+
+        if (op == GD_OP_STATUS_VOLUME)
+                return _gf_true;
+
+        if ((op == GD_OP_SET_VOLUME)) {
+                //check for set volume help
+                ret = dict_get_str (dict, "volname", &volname);
+                if (volname &&
+                    ((strcmp (volname, "help") == 0) ||
+                     (strcmp (volname, "help-xml") == 0))) {
+                        ret = dict_get_str (dict, "key1", &key);
+                        if (ret < 0)
+                                return _gf_true;
+                }
+        }
+
+        return _gf_false;
+}
+
+gf_boolean_t
+glusterd_is_op_quorum_validation_required (xlator_t *this, glusterd_op_t op,
+                                           dict_t *dict)
+{
+        gf_boolean_t    required = _gf_true;
+        char            *key = NULL;
+        char            *key_fixed = NULL;
+        int             ret = -1;
+
+        if (glusterd_is_get_op (this, op, dict)) {
+                required = _gf_false;
+                goto out;
+        }
+        if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
+                goto out;
+        if (op == GD_OP_SET_VOLUME)
+                ret = dict_get_str (dict, "key1", &key);
+        else if (op == GD_OP_RESET_VOLUME)
+                ret = dict_get_str (dict, "key", &key);
+        if (ret)
+                goto out;
+        ret = glusterd_check_option_exists (key, &key_fixed);
+        if (ret <= 0)
+                goto out;
+        if (key_fixed)
+                key = key_fixed;
+        if (glusterd_is_quorum_option (key))
+                required = _gf_false;
+out:
+        GF_FREE (key_fixed);
+        return required;
+}
+
+static int
+glusterd_op_validate_quorum (xlator_t *this, glusterd_op_t op,
+                             dict_t *dict, char **op_errstr)
+{
+        int                     ret = 0;
+        char                    *volname = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *errstr = NULL;
+
+
+        errstr = "Quorum not met. Volume operation not allowed.";
+        if (!glusterd_is_op_quorum_validation_required (this, op, dict))
+                goto out;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        if (does_gd_meet_server_quorum (this)) {
+                ret = 0;
+                goto out;
+        }
+
+        if (glusterd_is_volume_in_server_quorum (volinfo)) {
+                ret = -1;
+                *op_errstr = gf_strdup (errstr);
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+        rpc_clnt_procedure_t    *proc = NULL;
+        glusterd_conf_t         *priv = NULL;
+        xlator_t                *this = NULL;
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        dict_t                  *dict = NULL;
+        char                    *op_errstr  = NULL;
+        glusterd_op_t           op = GD_OP_NONE;
+        uint32_t                pending_count = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        op = glusterd_op_get_op ();
+
+        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                opinfo.op_errstr = op_errstr;
+                goto out;
+        }
+
+        ret = glusterd_op_validate_quorum (this, op, dict, &op_errstr);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+                opinfo.op_errstr = op_errstr;
+                goto out;
+        }
+
+        /* rsp_dict NULL from source */
+        ret = glusterd_op_stage_validate (op, dict, &op_errstr, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_STAGE_FAIL,
+                        gd_op_list[op], "localhost",
+                        (op_errstr) ? ":" : " ", (op_errstr) ? op_errstr : " ");
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_STAGE_FAIL,
+                                     "localhost");
+                opinfo.op_errstr = op_errstr;
+                goto out;
+        }
+
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                GF_ASSERT (peerinfo);
+
+                if (!peerinfo->connected || !peerinfo->mgmt)
+                        continue;
+                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+                        continue;
+
+                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_STAGE_OP];
+                GF_ASSERT (proc);
+                if (proc->fn) {
+                        ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                        "set peerinfo");
+                                goto out;
+                        }
+
+                        ret = proc->fn (NULL, this, dict);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
+                                        "send stage request for operation "
+                                        "'Volume %s' to peer %s",
+                                        gd_op_list[op], peerinfo->hostname);
+                                continue;
+                        }
+                        pending_count++;
+                }
+        }
+
+        opinfo.pending_count = pending_count;
+out:
+        if (dict)
+                dict_unref (dict);
+        if (ret) {
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             &event->txn_id, NULL);
+                opinfo.op_ret = ret;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent stage op request for "
+                "'Volume %s' to %d peers", gd_op_list[op],
+                opinfo.pending_count);
+
+        if (!opinfo.pending_count)
+                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+
+}
+
+static int32_t
+glusterd_op_start_rb_timer (dict_t *dict, uuid_t *txn_id)
+{
+        int32_t         op = 0;
+        struct timespec timeout = {0, };
+        glusterd_conf_t *priv = NULL;
+        int32_t         ret = -1;
+        dict_t          *rb_ctx = NULL;
+
+        GF_ASSERT (dict);
+        priv = THIS->private;
+
+        ret = dict_get_int32 (dict, "operation", &op);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "dict_get on operation failed");
+                goto out;
+        }
+
+        if (op != GF_REPLACE_OP_START) {
+                ret = glusterd_op_sm_inject_all_acc (txn_id);
+                goto out;
+        }
+
+        timeout.tv_sec  = 5;
+        timeout.tv_nsec = 0;
+
+
+        rb_ctx = dict_copy (dict, rb_ctx);
+        if (!rb_ctx) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't copy "
+                        "replace brick context. Can't start replace brick");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_bin (rb_ctx, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+                goto out;
+        } else
+               gf_log ("", GF_LOG_DEBUG,
+                        "transaction_id = %s", uuid_utoa (*txn_id));
+
+        priv->timer = gf_timer_call_after (THIS->ctx, timeout,
+                                           glusterd_do_replace_brick,
+                                           (void *) rb_ctx);
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+/* This function takes a dict and converts the uuid values of key specified
+ * into hostnames
+ */
+static int
+glusterd_op_volume_dict_uuid_to_hostname (dict_t *dict, const char *key_fmt,
+                                          int idx_min, int idx_max)
+{
+        int             ret = -1;
+        int             i = 0;
+        char            key[1024];
+        char            *uuid_str = NULL;
+        uuid_t          uuid = {0,};
+        char            *hostname = NULL;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key_fmt);
+
+        for (i = idx_min; i < idx_max; i++) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), key_fmt, i);
+                ret = dict_get_str (dict, key, &uuid_str);
+                if (ret)
+                        continue;
+
+                gf_log (this->name, GF_LOG_DEBUG, "Got uuid %s",
+                        uuid_str);
+
+                ret = uuid_parse (uuid_str, uuid);
+                /* if parsing fails don't error out
+                 * let the original value be retained
+                 */
+                if (ret)
+                        continue;
+
+                hostname = glusterd_uuid_to_hostname (uuid);
+                if (hostname) {
+                        gf_log (this->name, GF_LOG_DEBUG, "%s -> %s",
+                                uuid_str, hostname);
+                        ret = dict_set_dynstr (dict, key, hostname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Error setting hostname %s to dict",
+                                        hostname);
+                                GF_FREE (hostname);
+                                goto out;
+                        }
+                }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+reassign_defrag_status (dict_t *dict, char *key, gf_defrag_status_t *status)
+{
+        int ret = 0;
+
+        if (!*status)
+                return ret;
+
+        switch (*status) {
+        case GF_DEFRAG_STATUS_STARTED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
+                break;
+
+        case GF_DEFRAG_STATUS_STOPPED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED;
+                break;
+
+        case GF_DEFRAG_STATUS_COMPLETE:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
+                break;
+
+        case GF_DEFRAG_STATUS_FAILED:
+                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED;
+                break;
+        default:
+                break;
+         }
+
+        ret = dict_set_int32(dict, key, *status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to reset defrag %s in dict", key);
+
+        return ret;
+}
+
+/* Check and reassign the defrag_status enum got from the rebalance process
+ * of all peers so that the rebalance-status CLI command can display if a
+ * full-rebalance or just a fix-layout was carried out.
+ */
+static int
+glusterd_op_check_peer_defrag_status (dict_t *dict, int count)
+{
+        glusterd_volinfo_t *volinfo  = NULL;
+        gf_defrag_status_t status    = GF_DEFRAG_STATUS_NOT_STARTED;
+        char               key[256]  = {0,};
+        char               *volname  = NULL;
+        int                ret       = -1;
+        int                i         = 1;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        if (volinfo->rebal.defrag_cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                /* Fix layout was not issued; we don't need to reassign
+                   the status */
+                ret = 0;
+                goto out;
+        }
+
+        do {
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", i);
+                ret = dict_get_int32 (dict, key, (int32_t *)&status);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to get defrag %s", key);
+                        goto out;
+                }
+                ret = reassign_defrag_status (dict, key, &status);
+                if (ret)
+                        goto out;
+                i++;
+        } while (i <= count);
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+/* This function is used to modify the op_ctx dict before sending it back
+ * to cli. This is useful in situations like changing the peer uuids to
+ * hostnames etc.
+ */
+void
+glusterd_op_modify_op_ctx (glusterd_op_t op, void *ctx)
+{
+        int             ret = -1;
+        dict_t          *op_ctx = NULL;
+        int             brick_index_max = -1;
+        int             other_count = 0;
+        int             count = 0;
+        uint32_t        cmd = GF_CLI_STATUS_NONE;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (ctx)
+                op_ctx = ctx;
+        else
+                op_ctx = glusterd_op_get_ctx();
+
+        if (!op_ctx) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Operation context is not present.");
+                goto out;
+        }
+
+        switch (op) {
+        case GD_OP_STATUS_VOLUME:
+                ret = dict_get_uint32 (op_ctx, "cmd", &cmd);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get status cmd");
+                        goto out;
+                }
+                if (!(cmd & GF_CLI_STATUS_NFS || cmd & GF_CLI_STATUS_SHD ||
+                    (cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "op_ctx modification not required for status "
+                                "operation being performed");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (op_ctx, "brick-index-max",
+                                      &brick_index_max);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get brick-index-max");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (op_ctx, "other-count", &other_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get other-count");
+                        goto out;
+                }
+
+                count = brick_index_max + other_count + 1;
+
+                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
+                                                                "brick%d.path",
+                                                                0, count);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Failed uuid to hostname conversion");
+
+                break;
+
+        case GD_OP_PROFILE_VOLUME:
+                ret = dict_get_str_boolean (op_ctx, "nfs", _gf_false);
+                if (!ret)
+                        goto out;
+
+                ret = dict_get_int32 (op_ctx, "count", &count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get brick count");
+                        goto out;
+                }
+
+                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
+                                                                "%d-brick",
+                                                                1, (count + 1));
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Failed uuid to hostname conversion");
+
+                break;
+
+        /* For both rebalance and remove-brick status, the glusterd op is the
+         * same
+         */
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = dict_get_int32 (op_ctx, "count", &count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Failed to get count");
+                        goto out;
+                }
+
+                /* add 'node-name-%d' into op_ctx with value uuid_str.
+                   this will be used to convert to hostname later */
+                {
+                        char  key[1024];
+                        char *uuid_str = NULL;
+                        int   i;
+
+                        for (i = 1; i <= count; i++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "node-uuid-%d", i);
+                                ret = dict_get_str (op_ctx, key, &uuid_str);
+                                if (!ret) {
+                                        memset (key, 0, sizeof (key));
+                                        snprintf (key, sizeof (key),
+                                                  "node-name-%d", i);
+                                        ret = dict_set_str (op_ctx, key,
+                                                            uuid_str);
+                                }
+                        }
+                }
+
+                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
+                                                                "node-name-%d",
+                                                                1, (count + 1));
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Failed uuid to hostname conversion");
+
+                ret = glusterd_op_check_peer_defrag_status (op_ctx, count);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to reset defrag status for fix-layout");
+                break;
+
+        default:
+                ret = 0;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "op_ctx modification not required");
+                break;
+
+        }
+
+out:
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "op_ctx modification failed");
+        return;
+}
+
+static int
+glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_commit_hook_type_t type)
+{
+        glusterd_conf_t *priv                   = NULL;
+        char            hookdir[PATH_MAX]       = {0, };
+        char            scriptdir[PATH_MAX]     = {0, };
+        char            type_subdir[256]        = {0, };
+        char            *cmd_subdir             = NULL;
+        int             ret                     = -1;
+
+        priv = THIS->private;
+        switch (type) {
+                case GD_COMMIT_HOOK_NONE:
+                case GD_COMMIT_HOOK_MAX:
+                        /*Won't be called*/
+                        break;
+
+                case GD_COMMIT_HOOK_PRE:
+                        strcpy (type_subdir, "pre");
+                        break;
+                case GD_COMMIT_HOOK_POST:
+                        strcpy (type_subdir, "post");
+                        break;
+        }
+
+        cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
+        if (strlen (cmd_subdir) == 0)
+                return -1;
+
+        GLUSTERD_GET_HOOKS_DIR (hookdir, GLUSTERD_HOOK_VER, priv);
+        snprintf (scriptdir, sizeof (scriptdir), "%s/%s/%s",
+                  hookdir, cmd_subdir, type_subdir);
+
+        switch (type) {
+                case GD_COMMIT_HOOK_NONE:
+                case GD_COMMIT_HOOK_MAX:
+                        /*Won't be called*/
+                        break;
+
+                case GD_COMMIT_HOOK_PRE:
+                        ret = glusterd_hooks_run_hooks (scriptdir, op, op_ctx,
+                                                        type);
+                        break;
+                case GD_COMMIT_HOOK_POST:
+                        ret = glusterd_hooks_post_stub_enqueue (scriptdir, op,
+                                                                op_ctx);
+                        break;
+        }
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+        rpc_clnt_procedure_t    *proc = NULL;
+        glusterd_conf_t         *priv = NULL;
+        xlator_t                *this = NULL;
+        dict_t                  *dict = NULL;
+        dict_t                  *op_dict = NULL;
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        char                    *op_errstr  = NULL;
+        glusterd_op_t           op = GD_OP_NONE;
+        uint32_t                pending_count = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        op      = glusterd_op_get_op ();
+        op_dict = glusterd_op_get_ctx ();
+
+        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                opinfo.op_errstr = op_errstr;
+                goto out;
+        }
+
+        ret = glusterd_op_commit_perform (op, dict, &op_errstr, NULL); //rsp_dict invalid for source
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_COMMIT_FAIL,
+                        gd_op_list[op], "localhost", (op_errstr) ? ":" : " ",
+                        (op_errstr) ? op_errstr : " ");
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_COMMIT_FAIL,
+                                     "localhost");
+                opinfo.op_errstr = op_errstr;
+                goto out;
+        }
+
+
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                GF_ASSERT (peerinfo);
+
+                if (!peerinfo->connected || !peerinfo->mgmt)
+                        continue;
+                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+                        continue;
+
+                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_COMMIT_OP];
+                GF_ASSERT (proc);
+                if (proc->fn) {
+                        ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to set peerinfo");
+                                goto out;
+                        }
+                        ret = proc->fn (NULL, this, dict);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
+                                        "send commit request for operation "
+                                        "'Volume %s' to peer %s",
+                                        gd_op_list[op], peerinfo->hostname);
+                                continue;
+                        }
+                        pending_count++;
+                }
+        }
+
+        opinfo.pending_count = pending_count;
+        gf_log (this->name, GF_LOG_DEBUG, "Sent commit op req for 'Volume %s' "
+                "to %d peers", gd_op_list[op], opinfo.pending_count);
+out:
+        if (dict)
+                dict_unref (dict);
+        if (ret) {
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             &event->txn_id, NULL);
+                opinfo.op_ret = ret;
+        }
+
+        if (!opinfo.pending_count) {
+                if (op == GD_OP_REPLACE_BRICK) {
+                        ret = glusterd_op_start_rb_timer (op_dict,
+                                                          &event->txn_id);
+
+                } else {
+                        glusterd_op_modify_op_ctx (op, NULL);
+                        ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
+                }
+                goto err;
+        }
+
+err:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+
+}
+
+static int
+glusterd_op_ac_rcvd_stage_op_acc (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+
+        GF_ASSERT (event);
+
+        if (opinfo.pending_count > 0)
+                opinfo.pending_count--;
+
+        if (opinfo.pending_count > 0)
+                goto out;
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_ACC,
+                                           &event->txn_id, NULL);
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_stage_op_failed (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+
+        GF_ASSERT (event);
+
+        if (opinfo.pending_count > 0)
+                opinfo.pending_count--;
+
+        if (opinfo.pending_count > 0)
+                goto out;
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, NULL);
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_commit_op_failed (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+
+        GF_ASSERT (event);
+
+        if (opinfo.pending_count > 0)
+                opinfo.pending_count--;
+
+        if (opinfo.pending_count > 0)
+                goto out;
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, NULL);
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+        glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
+        gf_boolean_t                free_errstr = _gf_false;
+        xlator_t                    *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (event);
+        GF_ASSERT (ctx);
+        ev_ctx = ctx;
+
+        ret = glusterd_remove_pending_entry (&opinfo.pending_bricks, ev_ctx->pending_node->node);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unknown response received ");
+                ret = -1;
+                free_errstr = _gf_true;
+                goto out;
+        }
+        if (opinfo.brick_pending_count > 0)
+                opinfo.brick_pending_count--;
+        if (opinfo.op_ret == 0)
+                opinfo.op_ret = ev_ctx->op_ret;
+
+        if (opinfo.op_errstr == NULL)
+                opinfo.op_errstr = ev_ctx->op_errstr;
+        else
+                free_errstr = _gf_true;
+
+        if (opinfo.brick_pending_count > 0)
+                goto out;
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                           &event->txn_id, ev_ctx->commit_ctx);
+
+out:
+        if (ev_ctx->rsp_dict)
+                dict_unref (ev_ctx->rsp_dict);
+        if (free_errstr && ev_ctx->op_errstr)
+                GF_FREE (ev_ctx->op_errstr);
+        GF_FREE (ctx);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
+{
+        dict_t                 *op_ctx            = NULL;
+        int                     ret               = 0;
+        gf_boolean_t            commit_ack_inject = _gf_true;
+        glusterd_op_t           op                = GD_OP_NONE;
+        xlator_t               *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        op = glusterd_op_get_op ();
+        GF_ASSERT (event);
+
+        if (opinfo.pending_count > 0)
+                opinfo.pending_count--;
+
+        if (opinfo.pending_count > 0)
+                goto out;
+
+        if (op == GD_OP_REPLACE_BRICK) {
+                op_ctx = glusterd_op_get_ctx ();
+                if (!op_ctx) {
+                        gf_log (this->name, GF_LOG_CRITICAL, "Operation "
+                                "context is not present.");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_op_start_rb_timer (op_ctx, &event->txn_id);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Couldn't start "
+                                "replace-brick operation.");
+                        goto out;
+                }
+
+                commit_ack_inject = _gf_false;
+                goto out;
+        }
+
+
+out:
+        if (commit_ack_inject) {
+                if (ret)
+                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                                           &event->txn_id, NULL);
+                else if (!opinfo.pending_count) {
+                        glusterd_op_modify_op_ctx (op, NULL);
+                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC,
+                                                           &event->txn_id, NULL);
+                }
+                /*else do nothing*/
+        }
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_rcvd_unlock_acc (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+
+        GF_ASSERT (event);
+
+        if (opinfo.pending_count > 0)
+                opinfo.pending_count--;
+
+        if (opinfo.pending_count > 0)
+                goto out;
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
+                                           &event->txn_id, NULL);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_op_clear_errstr() {
+        opinfo.op_errstr = NULL;
+        return 0;
+}
+
+int32_t
+glusterd_op_set_ctx (void *ctx)
+{
+
+        opinfo.op_ctx = ctx;
+
+        return 0;
+
+}
+
+int32_t
+glusterd_op_reset_ctx ()
+{
+
+        glusterd_op_set_ctx (NULL);
+
+        return 0;
+}
+
+int32_t
+glusterd_op_txn_complete (uuid_t *txn_id)
+{
+        int32_t                 ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        int32_t                 op = -1;
+        int32_t                 op_ret = 0;
+        int32_t                 op_errno = 0;
+        rpcsvc_request_t        *req = NULL;
+        void                    *ctx = NULL;
+        char                    *op_errstr = NULL;
+        char                    *volname = NULL;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        op  = glusterd_op_get_op ();
+        ctx = glusterd_op_get_ctx ();
+        op_ret = opinfo.op_ret;
+        op_errno = opinfo.op_errno;
+        req = opinfo.req;
+        if (opinfo.op_errstr)
+                op_errstr = opinfo.op_errstr;
+
+        opinfo.op_ret = 0;
+        opinfo.op_errno = 0;
+        glusterd_op_clear_op ();
+        glusterd_op_reset_ctx ();
+        glusterd_op_clear_errstr ();
+
+        /* Based on the op-version, we release the cluster or mgmt_v3 lock */
+        if (priv->op_version < 3) {
+                ret = glusterd_unlock (MY_UUID);
+                /* unlock cant/shouldnt fail here!! */
+                if (ret)
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to clear local lock, ret: %d", ret);
+                else
+                        gf_log (this->name, GF_LOG_DEBUG, "Cleared local lock");
+        } else {
+                ret = dict_get_str (ctx, "volname", &volname);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire volname");
+
+                if (volname) {
+                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                }
+        }
+
+        ret = glusterd_op_send_cli_response (op, op_ret,
+                                             op_errno, req, ctx, op_errstr);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Responding to cli failed, "
+                        "ret: %d", ret);
+                //Ignore this error, else state machine blocks
+                ret = 0;
+        }
+
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+
+        if (priv->pending_quorum_action)
+                glusterd_do_quorum_action ();
+
+        /* Clearing the transaction opinfo */
+        ret = glusterd_clear_txn_opinfo (txn_id);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to clear transaction's opinfo");
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_op_ac_unlocked_all (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+
+        GF_ASSERT (event);
+
+        ret = glusterd_op_txn_complete (&event->txn_id);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = -1;
+        glusterd_req_ctx_t      *req_ctx = NULL;
+        int32_t                 status = 0;
+        dict_t                  *rsp_dict  = NULL;
+        char                    *op_errstr = NULL;
+        dict_t                  *dict = NULL;
+        xlator_t                *this = NULL;
+        uuid_t                  *txn_id = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (ctx);
+
+        req_ctx = ctx;
+
+        dict = req_ctx->dict;
+
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get new dictionary");
+                return -1;
+        }
+
+        status = glusterd_op_stage_validate (req_ctx->op, dict, &op_errstr,
+                                             rsp_dict);
+
+        if (status) {
+                gf_log (this->name, GF_LOG_ERROR, "Stage failed on operation"
+                        " 'Volume %s', Status : %d", gd_op_list[req_ctx->op],
+                        status);
+        }
+
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+
+        if (txn_id)
+                uuid_copy (*txn_id, event->txn_id);
+        else
+                gf_log (this->name, GF_LOG_ERROR, "Out of Memory");
+
+        ret = dict_set_bin (rsp_dict, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+
+        ret = glusterd_op_stage_send_resp (req_ctx->req, req_ctx->op,
+                                           status, op_errstr, rsp_dict);
+
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        return ret;
+}
+
+static gf_boolean_t
+glusterd_need_brick_op (glusterd_op_t op)
+{
+        gf_boolean_t ret        = _gf_false;
+
+        GF_ASSERT (GD_OP_NONE < op && op < GD_OP_MAX);
+
+        switch (op) {
+        case GD_OP_PROFILE_VOLUME:
+        case GD_OP_STATUS_VOLUME:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+        case GD_OP_HEAL_VOLUME:
+                ret = _gf_true;
+                break;
+        default:
+                ret = _gf_false;
+        }
+
+        return ret;
+}
+
+dict_t*
+glusterd_op_init_commit_rsp_dict (glusterd_op_t op)
+{
+        dict_t                  *rsp_dict = NULL;
+        dict_t                  *op_ctx   = NULL;
+
+        GF_ASSERT (GD_OP_NONE < op && op < GD_OP_MAX);
+
+        if (glusterd_need_brick_op (op)) {
+                op_ctx = glusterd_op_get_ctx ();
+                GF_ASSERT (op_ctx);
+                rsp_dict = dict_ref (op_ctx);
+        } else {
+                rsp_dict = dict_new ();
+        }
+
+        return rsp_dict;
+}
+
+static int
+glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                       ret        = 0;
+        glusterd_req_ctx_t       *req_ctx    = NULL;
+        int32_t                   status     = 0;
+        char                     *op_errstr  = NULL;
+        dict_t                   *dict       = NULL;
+        dict_t                   *rsp_dict   = NULL;
+        xlator_t                 *this       = NULL;
+        uuid_t                   *txn_id     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (ctx);
+
+        req_ctx = ctx;
+
+        dict = req_ctx->dict;
+
+        rsp_dict = glusterd_op_init_commit_rsp_dict (req_ctx->op);
+        if (NULL == rsp_dict)
+                return -1;
+
+
+        if (GD_OP_CLEARLOCKS_VOLUME == req_ctx->op) {
+                /*clear locks should be run only on
+                 * originator glusterd*/
+                status = 0;
+
+        } else {
+                status = glusterd_op_commit_perform (req_ctx->op, dict,
+                                                     &op_errstr, rsp_dict);
+        }
+
+        if (status)
+                gf_log (this->name, GF_LOG_ERROR, "Commit of operation "
+                        "'Volume %s' failed: %d", gd_op_list[req_ctx->op],
+                        status);
+
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+
+        if (txn_id)
+                uuid_copy (*txn_id, event->txn_id);
+        else
+                gf_log (this->name, GF_LOG_ERROR, "Out of Memory");
+
+        ret = dict_set_bin (rsp_dict, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to set transaction id.");
+
+        ret = glusterd_op_commit_send_resp (req_ctx->req, req_ctx->op,
+                                            status, op_errstr, rsp_dict);
+
+        if (op_errstr && (strcmp (op_errstr, "")))
+                GF_FREE (op_errstr);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_op_ac_send_commit_failed (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                             ret = 0;
+        glusterd_req_ctx_t              *req_ctx = NULL;
+        dict_t                          *op_ctx = NULL;
+
+        GF_ASSERT (ctx);
+
+        req_ctx = ctx;
+
+        op_ctx = glusterd_op_get_ctx ();
+
+        ret = glusterd_op_commit_send_resp (req_ctx->req, req_ctx->op,
+                                            opinfo.op_ret, opinfo.op_errstr,
+                                            op_ctx);
+
+        if (opinfo.op_errstr && (strcmp (opinfo.op_errstr, ""))) {
+                GF_FREE (opinfo.op_errstr);
+                opinfo.op_errstr = NULL;
+        }
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+static int
+glusterd_op_sm_transition_state (glusterd_op_info_t *opinfo,
+                                 glusterd_op_sm_t *state,
+                                 glusterd_op_sm_event_type_t event_type)
+{
+        glusterd_conf_t         *conf = NULL;
+
+        GF_ASSERT (state);
+        GF_ASSERT (opinfo);
+
+        conf = THIS->private;
+        GF_ASSERT (conf);
+
+        (void) glusterd_sm_tr_log_transition_add (&conf->op_sm_log,
+                                           opinfo->state.state,
+                                           state[event_type].next_state,
+                                           event_type);
+
+        opinfo->state.state = state[event_type].next_state;
+        return 0;
+}
+
+int32_t
+glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
+                            dict_t *rsp_dict)
+{
+        int ret = -1;
+        xlator_t *this = THIS;
+
+        switch (op) {
+                case GD_OP_CREATE_VOLUME:
+                        ret = glusterd_op_stage_create_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_START_VOLUME:
+                        ret = glusterd_op_stage_start_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_STOP_VOLUME:
+                        ret = glusterd_op_stage_stop_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_DELETE_VOLUME:
+                        ret = glusterd_op_stage_delete_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_ADD_BRICK:
+                        ret = glusterd_op_stage_add_brick (dict, op_errstr);
+                        break;
+
+                case GD_OP_REPLACE_BRICK:
+                        ret = glusterd_op_stage_replace_brick (dict, op_errstr,
+                                                               rsp_dict);
+                        break;
+
+                case GD_OP_SET_VOLUME:
+                        ret = glusterd_op_stage_set_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_RESET_VOLUME:
+                        ret = glusterd_op_stage_reset_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_REMOVE_BRICK:
+                        ret = glusterd_op_stage_remove_brick (dict, op_errstr);
+                        break;
+
+                case GD_OP_LOG_ROTATE:
+                        ret = glusterd_op_stage_log_rotate (dict, op_errstr);
+                        break;
+
+                case GD_OP_SYNC_VOLUME:
+                        ret = glusterd_op_stage_sync_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_GSYNC_CREATE:
+                        ret = glusterd_op_stage_gsync_create (dict, op_errstr);
+                        break;
+
+                case GD_OP_GSYNC_SET:
+                        ret = glusterd_op_stage_gsync_set (dict, op_errstr);
+                        break;
+
+                case GD_OP_PROFILE_VOLUME:
+                        ret = glusterd_op_stage_stats_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_QUOTA:
+                        ret = glusterd_op_stage_quota (dict, op_errstr);
+                        break;
+
+                case GD_OP_STATUS_VOLUME:
+                        ret = glusterd_op_stage_status_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_REBALANCE:
+                case GD_OP_DEFRAG_BRICK_VOLUME:
+                        ret = glusterd_op_stage_rebalance (dict, op_errstr);
+                        break;
+
+                case GD_OP_HEAL_VOLUME:
+                        ret = glusterd_op_stage_heal_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_STATEDUMP_VOLUME:
+                        ret = glusterd_op_stage_statedump_volume (dict,
+                                                                  op_errstr);
+                        break;
+                case GD_OP_CLEARLOCKS_VOLUME:
+                        ret = glusterd_op_stage_clearlocks_volume (dict,
+                                                                   op_errstr);
+                        break;
+
+                case GD_OP_COPY_FILE:
+                        ret = glusterd_op_stage_copy_file (dict, op_errstr);
+                        break;
+
+                case GD_OP_SYS_EXEC:
+                        ret = glusterd_op_stage_sys_exec (dict, op_errstr);
+                        break;
+
+                default:
+                        gf_log (this->name, GF_LOG_ERROR, "Unknown op %s",
+                                gd_op_list[op]);
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
+        return ret;
+}
+
+
+int32_t
+glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
+                            dict_t *rsp_dict)
+{
+        int ret = -1;
+        xlator_t *this = THIS;
+
+        glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_PRE);
+        switch (op) {
+                case GD_OP_CREATE_VOLUME:
+                        ret = glusterd_op_create_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_START_VOLUME:
+                        ret = glusterd_op_start_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_STOP_VOLUME:
+                        ret = glusterd_op_stop_volume (dict);
+                        break;
+
+                case GD_OP_DELETE_VOLUME:
+                        ret = glusterd_op_delete_volume (dict);
+                        break;
+
+                case GD_OP_ADD_BRICK:
+                        ret = glusterd_op_add_brick (dict, op_errstr);
+                        break;
+
+                case GD_OP_REPLACE_BRICK:
+                        ret = glusterd_op_replace_brick (dict, rsp_dict);
+                        break;
+
+                case GD_OP_SET_VOLUME:
+                        ret = glusterd_op_set_volume (dict);
+                        break;
+
+                case GD_OP_RESET_VOLUME:
+                        ret = glusterd_op_reset_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_REMOVE_BRICK:
+                        ret = glusterd_op_remove_brick (dict, op_errstr);
+                        break;
+
+                case GD_OP_LOG_ROTATE:
+                        ret = glusterd_op_log_rotate (dict);
+                        break;
+
+                case GD_OP_SYNC_VOLUME:
+                        ret = glusterd_op_sync_volume (dict, op_errstr, rsp_dict);
+                        break;
+
+                case GD_OP_GSYNC_CREATE:
+                        ret = glusterd_op_gsync_create (dict, op_errstr,
+                                                        rsp_dict);
+                        break;
+
+                case GD_OP_GSYNC_SET:
+                        ret = glusterd_op_gsync_set (dict, op_errstr, rsp_dict);
+                        break;
+
+                case GD_OP_PROFILE_VOLUME:
+                        ret = glusterd_op_stats_volume (dict, op_errstr,
+                                                        rsp_dict);
+                        break;
+
+                case GD_OP_QUOTA:
+                        ret = glusterd_op_quota (dict, op_errstr, rsp_dict);
+                        break;
+
+                case GD_OP_STATUS_VOLUME:
+                        ret = glusterd_op_status_volume (dict, op_errstr, rsp_dict);
+                        break;
+
+                case GD_OP_REBALANCE:
+                case GD_OP_DEFRAG_BRICK_VOLUME:
+                        ret = glusterd_op_rebalance (dict, op_errstr, rsp_dict);
+                        break;
+
+                case GD_OP_HEAL_VOLUME:
+                        ret = glusterd_op_heal_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_STATEDUMP_VOLUME:
+                        ret = glusterd_op_statedump_volume (dict, op_errstr);
+                        break;
+
+                case GD_OP_CLEARLOCKS_VOLUME:
+                        ret = glusterd_op_clearlocks_volume (dict, op_errstr,
+                                                             rsp_dict);
+                        break;
+
+                case GD_OP_COPY_FILE:
+                        ret = glusterd_op_copy_file (dict, op_errstr);
+                        break;
+
+                case GD_OP_SYS_EXEC:
+                        ret = glusterd_op_sys_exec (dict, op_errstr, rsp_dict);
+                        break;
+
+                default:
+                        gf_log (this->name, GF_LOG_ERROR, "Unknown op %s",
+                                gd_op_list[op]);
+                        break;
+        }
+
+        if (ret == 0)
+            glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_POST);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+static int
+glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr,
+                                    struct list_head *selected)
+{
+        int                                     ret = 0;
+        int                                     flags = 0;
+        char                                    *volname = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        glusterd_brickinfo_t                    *brickinfo = NULL;
+        glusterd_pending_node_t                 *pending_node = NULL;
+
+        ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
+        if (ret)
+                goto out;
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (glusterd_is_brick_started (brickinfo)) {
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                goto out;
+                        } else {
+                                pending_node->node = brickinfo;
+                                pending_node->type = GD_NODE_BRICK;
+                                list_add_tail (&pending_node->list, selected);
+                                pending_node = NULL;
+                        }
+                }
+        }
+
+out:
+        return ret;
+}
+
+static int
+glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
+                                     struct list_head *selected)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        glusterd_brickinfo_t                    *brickinfo = NULL;
+        char                                    *brick = NULL;
+        int32_t                                 count = 0;
+        int32_t                                 i = 1;
+        char                                    key[256] = {0,};
+        glusterd_pending_node_t                 *pending_node = NULL;
+        int32_t                                 force = 0;
+
+
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get count");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "force", &force);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_INFO, "force flag is not set");
+                ret = 0;
+                goto out;
+        }
+
+        while ( i <= count) {
+                snprintf (key, 256, "brick%d", i);
+                ret = dict_get_str (dict, key, &brick);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Unable to get brick");
+                        goto out;
+                }
+
+                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                              &brickinfo);
+                if (ret)
+                        goto out;
+                if (glusterd_is_brick_started (brickinfo)) {
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                goto out;
+                        } else {
+                                pending_node->node = brickinfo;
+                                pending_node->type = GD_NODE_BRICK;
+                                list_add_tail (&pending_node->list, selected);
+                                pending_node = NULL;
+                        }
+                }
+                i++;
+        }
+
+out:
+        return ret;
+}
+
+static int
+glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr,
+                                       struct list_head *selected)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        char                                    msg[2048] = {0,};
+        glusterd_conf_t                         *priv = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        xlator_t                                *this = NULL;
+        int32_t                                 stats_op = GF_CLI_STATS_NONE;
+        glusterd_brickinfo_t                    *brickinfo = NULL;
+        glusterd_pending_node_t                 *pending_node = NULL;
+        char                                    *brick = NULL;
+
+
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume %s does not exists",
+                          volname);
+
+                *op_errstr = gf_strdup (msg);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "op", &stats_op);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "volume profile op get failed");
+                goto out;
+        }
+
+        switch (stats_op) {
+        case GF_CLI_STATS_START:
+        case GF_CLI_STATS_STOP:
+                goto out;
+                break;
+        case GF_CLI_STATS_INFO:
+                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+                if (ret) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "NFS server"
+                                        " is not running");
+                                goto out;
+                        }
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                goto out;
+                        }
+                        pending_node->node = priv->nfs;
+                        pending_node->type = GD_NODE_NFS;
+                        list_add_tail (&pending_node->list, selected);
+                        pending_node = NULL;
+
+                        ret = 0;
+                        goto out;
+
+                }
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        if (glusterd_is_brick_started (brickinfo)) {
+                                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                          gf_gld_mt_pending_node_t);
+                                if (!pending_node) {
+                                        ret = -1;
+                                        goto out;
+                                } else {
+                                        pending_node->node = brickinfo;
+                                        pending_node->type = GD_NODE_BRICK;
+                                        list_add_tail (&pending_node->list,
+                                                       selected);
+                                        pending_node = NULL;
+                                }
+                        }
+                }
+                break;
+
+        case GF_CLI_STATS_TOP:
+                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
+                if (ret) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "NFS server"
+                                        " is not running");
+                                goto out;
+                        }
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                goto out;
+                        }
+                        pending_node->node = priv->nfs;
+                        pending_node->type = GD_NODE_NFS;
+                        list_add_tail (&pending_node->list, selected);
+                        pending_node = NULL;
+
+                        ret = 0;
+                        goto out;
+
+                }
+                ret = dict_get_str (dict, "brick", &brick);
+                if (!ret) {
+                        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
+                                                                      &brickinfo);
+                        if (ret)
+                                goto out;
+
+                        if (!glusterd_is_brick_started (brickinfo))
+                                goto out;
+
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                goto out;
+                        } else {
+                                pending_node->node = brickinfo;
+                                pending_node->type = GD_NODE_BRICK;
+                                list_add_tail (&pending_node->list,
+                                               selected);
+                                pending_node = NULL;
+                                goto out;
+                        }
+                }
+                ret = 0;
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        if (glusterd_is_brick_started (brickinfo)) {
+                                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                          gf_gld_mt_pending_node_t);
+                                if (!pending_node) {
+                                        ret = -1;
+                                        goto out;
+                                } else {
+                                        pending_node->node = brickinfo;
+                                        pending_node->type = GD_NODE_BRICK;
+                                        list_add_tail (&pending_node->list,
+                                                       selected);
+                                        pending_node = NULL;
+                                }
+                        }
+                }
+                break;
+
+        default:
+                GF_ASSERT (0);
+                gf_log ("glusterd", GF_LOG_ERROR, "Invalid profile op: %d",
+                        stats_op);
+                ret = -1;
+                goto out;
+                break;
+        }
+
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+_add_rxlator_to_dict (dict_t *dict, char *volname, int index, int count)
+{
+        int     ret             = -1;
+        char    key[128]        = {0,};
+        char    *xname          = NULL;
+
+        snprintf (key, sizeof (key), "xl-%d", count);
+        ret = gf_asprintf (&xname, "%s-replicate-%d", volname, index);
+        if (ret == -1)
+                goto out;
+
+        ret = dict_set_dynstr (dict, key, xname);
+        if (ret)
+                goto out;
+
+        ret = dict_set_int32 (dict, xname, index);
+out:
+        return ret;
+}
+
+int
+get_replica_index_for_per_replica_cmd (glusterd_volinfo_t *volinfo,
+                                       dict_t *dict) {
+        int                     ret = 0;
+        char                    *hostname = NULL;
+        char                    *path = NULL;
+        int                     index = 0;
+        glusterd_brickinfo_t   *brickinfo = NULL;
+        int                     cmd_replica_index = -1;
+        int                     replica_count = -1;
+
+
+        if (!dict)  {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "per-replica-cmd-hostname", &hostname);
+        if (ret)
+                goto out;
+        ret = dict_get_str (dict, "per-replica-cmd-path", &path);
+        if (ret)
+                goto out;
+
+        replica_count = volinfo->replica_count;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+                if (!strcmp (brickinfo->path,  path) &&
+                    !strcmp (brickinfo->hostname, hostname)) {
+                        cmd_replica_index = index/(replica_count);
+                        goto out;
+                }
+                index++;
+        }
+
+
+out:
+        if (ret)
+                cmd_replica_index = -1;
+
+        return cmd_replica_index;
+}
+
+int
+_select_rxlators_with_local_bricks (xlator_t *this, glusterd_volinfo_t *volinfo,
+                                    dict_t *dict, cli_cmd_type type)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_conf_t         *priv   = NULL;
+        int                     index = 0;
+        int                     rxlator_count = 0;
+        int                     replica_count = 0;
+        gf_boolean_t            add     = _gf_false;
+        int                     ret = 0;
+        int                     cmd_replica_index = -1;
+
+        priv = this->private;
+        replica_count = volinfo->replica_count;
+
+        if (type == PER_REPLICA) {
+
+                cmd_replica_index = get_replica_index_for_per_replica_cmd
+                                    (volinfo, dict);
+                if (cmd_replica_index == -1) {
+                        ret = -1;
+                        goto err;
+                }
+        }
+
+        index = 1;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+
+                switch (type) {
+                        case ALL_REPLICA:
+                                if (!uuid_compare (MY_UUID, brickinfo->uuid))
+                                        add = _gf_true;
+                                break;
+                        case PER_REPLICA:
+                                if (!uuid_compare (MY_UUID, brickinfo->uuid) &&
+                                 ((index-1)/replica_count == cmd_replica_index))
+
+                                                add = _gf_true;
+                                break;
+                }
+
+                if (index % replica_count == 0) {
+                        if (add) {
+                                _add_rxlator_to_dict (dict, volinfo->volname,
+                                                      (index-1)/replica_count,
+                                                      rxlator_count);
+                                rxlator_count++;
+                        }
+                        add = _gf_false;
+                }
+
+                index++;
+        }
+err:
+        if (ret)
+                rxlator_count = -1;
+
+        return rxlator_count;
+}
+
+int
+_select_rxlators_for_full_self_heal (xlator_t *this,
+                                     glusterd_volinfo_t *volinfo,
+                                     dict_t *dict)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_conf_t         *priv   = NULL;
+        int                     index = 1;
+        int                     rxlator_count = 0;
+        int                     replica_count = 0;
+        uuid_t                  candidate = {0};
+
+        priv = this->private;
+        replica_count = volinfo->replica_count;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+
+                if (uuid_compare (brickinfo->uuid, candidate) > 0)
+                        uuid_copy (candidate, brickinfo->uuid);
+
+                if (index % replica_count == 0) {
+                        if (!uuid_compare (MY_UUID, candidate)) {
+                                _add_rxlator_to_dict (dict, volinfo->volname,
+                                                      (index-1)/replica_count,
+                                                      rxlator_count);
+                                rxlator_count++;
+                        }
+                        uuid_clear (candidate);
+                }
+
+                index++;
+        }
+        return rxlator_count;
+}
+
+
+static int
+glusterd_bricks_select_snap (dict_t *dict, char **op_errstr,
+                             struct list_head *selected)
+{
+        int                        ret           = -1;
+        glusterd_conf_t            *priv         = NULL;
+        xlator_t                   *this         = NULL;
+        glusterd_pending_node_t    *pending_node = NULL;
+        glusterd_volinfo_t         *volinfo      = NULL;
+        char                       *volname      = NULL;
+        glusterd_brickinfo_t       *brickinfo    = NULL;
+        int                         brick_index  = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get"
+                        " volname");
+                goto out;
+        }
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret)
+                goto out;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                brick_index++;
+                if (uuid_compare (brickinfo->uuid, MY_UUID) ||
+                    !glusterd_is_brick_started (brickinfo)) {
+                        continue;
+                }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = brickinfo;
+                pending_node->type = GD_NODE_BRICK;
+                pending_node->index = brick_index;
+                list_add_tail (&pending_node->list,
+                               selected);
+                pending_node = NULL;
+        }
+
+        ret = 0;
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning ret %d", ret);
+        return ret;
+}
+
+static int
+fill_shd_status_for_local_bricks (dict_t *dict, glusterd_volinfo_t *volinfo,
+                                  cli_cmd_type type, dict_t *req_dict)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        char                    msg[1024] = {0,};
+        char                    key[1024]  = {0,};
+        char                    value[1024] = {0,};
+        int                     index = 0;
+        int                     ret = 0;
+        xlator_t               *this = NULL;
+        int                     cmd_replica_index = -1;
+
+        this = THIS;
+        snprintf (msg, sizeof (msg), "self-heal-daemon is not running on");
+
+        if (type == PER_REPLICA) {
+                cmd_replica_index = get_replica_index_for_per_replica_cmd
+                                    (volinfo, req_dict);
+                if (cmd_replica_index == -1) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Could not find the "
+                                "replica index for per replica type command");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_is_null (brickinfo->uuid))
+                        (void)glusterd_resolve_brick (brickinfo);
+
+                if (uuid_compare (MY_UUID, brickinfo->uuid)) {
+                        index++;
+                        continue;
+                }
+
+                if (type == PER_REPLICA) {
+                      if (cmd_replica_index != (index/volinfo->replica_count)) {
+                              index++;
+                              continue;
+                        }
+
+                }
+                snprintf (key, sizeof (key), "%d-status",index);
+                snprintf (value, sizeof (value), "%s %s",msg,
+                          uuid_utoa(MY_UUID));
+                ret = dict_set_dynstr (dict, key, gf_strdup(value));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to"
+                                "set the dictionary for shd status msg");
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "%d-shd-status",index);
+                ret = dict_set_str (dict, key, "off");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to"
+                                " set dictionary for shd status msg");
+                        goto out;
+                }
+
+                index++;
+        }
+
+out:
+        return ret;
+
+}
+
+
+static int
+glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr,
+                                    struct list_head *selected,
+                                    dict_t *rsp_dict)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        glusterd_conf_t                         *priv = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        xlator_t                                *this = NULL;
+        char                                    msg[2048] = {0,};
+        glusterd_pending_node_t                 *pending_node = NULL;
+        gf_xl_afr_op_t                          heal_op = GF_AFR_OP_INVALID;
+        int                                     rxlator_count = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume %s does not exist",
+                          volname);
+
+                *op_errstr = gf_strdup (msg);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
+        if (ret || (heal_op == GF_AFR_OP_INVALID)) {
+                gf_log ("glusterd", GF_LOG_ERROR, "heal op invalid");
+                goto out;
+        }
+
+        switch (heal_op) {
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        if (!rsp_dict) {
+                                gf_log (this->name, GF_LOG_ERROR, "Received "
+                                        "empty ctx.");
+                                goto out;
+                        }
+
+                        ret = fill_shd_status_for_local_bricks (rsp_dict,
+                                                                volinfo,
+                                                                ALL_REPLICA,
+                                                                dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "fill the shd status for the local "
+                                        "bricks");
+                        goto out;
+
+                }
+                break;
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        if (!rsp_dict) {
+                                gf_log (this->name, GF_LOG_ERROR, "Received "
+                                        "empty ctx.");
+                                goto out;
+                        }
+                        ret = fill_shd_status_for_local_bricks (rsp_dict,
+                                                                volinfo,
+                                                                PER_REPLICA,
+                                                                dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "fill the shd status for the local"
+                                        " bricks.");
+                        goto out;
+
+                }
+                break;
+                default:
+                        break;
+        }
+
+
+        switch (heal_op) {
+        case GF_AFR_OP_HEAL_FULL:
+                rxlator_count = _select_rxlators_for_full_self_heal (this,
+                                                                     volinfo,
+                                                                     dict);
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                rxlator_count = _select_rxlators_with_local_bricks (this,
+                                                                   volinfo,
+                                                                   dict,
+                                                                   PER_REPLICA);
+                break;
+        default:
+                rxlator_count = _select_rxlators_with_local_bricks (this,
+                                                                    volinfo,
+                                                                    dict,
+                                                                   ALL_REPLICA);
+                break;
+        }
+        if (!rxlator_count)
+                goto out;
+        if (rxlator_count == -1){
+                gf_log (this->name, GF_LOG_ERROR, "Could not determine the"
+                        "translator count");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "count", rxlator_count);
+        if (ret)
+                goto out;
+
+        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                  gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+                ret = -1;
+                goto out;
+        } else {
+                pending_node->node = priv->shd;
+                pending_node->type = GD_NODE_SHD;
+                list_add_tail (&pending_node->list, selected);
+                pending_node = NULL;
+        }
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning ret %d", ret);
+        return ret;
+
+}
+
+static int
+glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
+                                         struct list_head *selected)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        xlator_t                                *this = NULL;
+        char                                    msg[2048] = {0,};
+        glusterd_pending_node_t                 *pending_node = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume %s does not exist",
+                          volname);
+
+                *op_errstr = gf_strdup (msg);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                  gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+                ret = -1;
+                goto out;
+        } else {
+                pending_node->node = volinfo;
+                pending_node->type = GD_NODE_REBALANCE;
+                list_add_tail (&pending_node->list,
+                               &opinfo.pending_bricks);
+                pending_node = NULL;
+        }
+
+out:
+        return ret;
+}
+
+
+
+
+static int
+glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr,
+                                      struct list_head *selected)
+{
+        int                     ret = -1;
+        int                     cmd = 0;
+        int                     brick_index = -1;
+        char                    *volname = NULL;
+        char                    *brickname = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_pending_node_t *pending_node = NULL;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+
+        GF_ASSERT (dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get status type");
+                goto out;
+        }
+
+        if (cmd & GF_CLI_STATUS_ALL)
+                goto out;
+
+        switch (cmd & GF_CLI_STATUS_MASK) {
+        case GF_CLI_STATUS_MEM:
+        case GF_CLI_STATUS_CLIENTS:
+        case GF_CLI_STATUS_INODE:
+        case GF_CLI_STATUS_FD:
+        case GF_CLI_STATUS_CALLPOOL:
+        case GF_CLI_STATUS_NFS:
+        case GF_CLI_STATUS_SHD:
+                break;
+        default:
+                goto out;
+        }
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volname");
+                goto out;
+        }
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                goto out;
+        }
+
+        if ( (cmd & GF_CLI_STATUS_BRICK) != 0) {
+                ret = dict_get_str (dict, "brick", &brickname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get brick");
+                        goto out;
+                }
+                ret = glusterd_volume_brickinfo_get_by_brick (brickname,
+                                                              volinfo,
+                                                              &brickinfo);
+                if (ret)
+                        goto out;
+
+                if (uuid_compare (brickinfo->uuid, MY_UUID)||
+                    !glusterd_is_brick_started (brickinfo))
+                        goto out;
+
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = brickinfo;
+                pending_node->type = GD_NODE_BRICK;
+                pending_node->index = 0;
+                list_add_tail (&pending_node->list, selected);
+
+                ret = 0;
+        } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+                if (!glusterd_is_nodesvc_online ("nfs")) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "NFS server is not running");
+                        goto out;
+                }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = priv->nfs;
+                pending_node->type = GD_NODE_NFS;
+                pending_node->index = 0;
+                list_add_tail (&pending_node->list, selected);
+
+                ret = 0;
+        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Self-heal daemon is not running");
+                        goto out;
+                }
+                pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                          gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                }
+                pending_node->node = priv->shd;
+                pending_node->type = GD_NODE_SHD;
+                pending_node->index = 0;
+                list_add_tail (&pending_node->list, selected);
+
+                ret = 0;
+        } else {
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        brick_index++;
+                        if (uuid_compare (brickinfo->uuid, MY_UUID) ||
+                            !glusterd_is_brick_started (brickinfo)) {
+                                continue;
+                        }
+                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
+                                                  gf_gld_mt_pending_node_t);
+                        if (!pending_node) {
+                                ret = -1;
+                                gf_log (THIS->name ,GF_LOG_ERROR,
+                                        "Unable to allocate memory");
+                                goto out;
+                        }
+                        pending_node->node = brickinfo;
+                        pending_node->type = GD_NODE_BRICK;
+                        pending_node->index = brick_index;
+                        list_add_tail (&pending_node->list, selected);
+                        pending_node = NULL;
+                }
+        }
+out:
+        return ret;
+}
+
+static int
+glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                             ret = 0;
+        rpc_clnt_procedure_t            *proc = NULL;
+        glusterd_conf_t                 *priv = NULL;
+        xlator_t                        *this = NULL;
+        glusterd_op_t                   op = GD_OP_NONE;
+        glusterd_req_ctx_t              *req_ctx = NULL;
+        char                            *op_errstr = NULL;
+
+        this = THIS;
+        priv = this->private;
+
+        if (ctx) {
+                req_ctx = ctx;
+        } else {
+                req_ctx = GF_CALLOC (1, sizeof (*req_ctx),
+                                     gf_gld_mt_op_allack_ctx_t);
+                op = glusterd_op_get_op ();
+                req_ctx->op = op;
+                uuid_copy (req_ctx->uuid, MY_UUID);
+                ret = glusterd_op_build_payload (&req_ctx->dict, &op_errstr,
+                                                 NULL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                                gd_op_list[op]);
+                        if (op_errstr == NULL)
+                                gf_asprintf (&op_errstr,
+                                             OPERRSTR_BUILD_PAYLOAD);
+                        opinfo.op_errstr = op_errstr;
+                        goto out;
+                }
+        }
+
+        proc = &priv->gfs_mgmt->proctable[GLUSTERD_BRICK_OP];
+        if (proc->fn) {
+                ret = proc->fn (NULL, this, req_ctx);
+                if (ret)
+                        goto out;
+        }
+
+        if (!opinfo.pending_count && !opinfo.brick_pending_count) {
+                glusterd_clear_pending_nodes (&opinfo.pending_bricks);
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
+                                                   &event->txn_id, req_ctx);
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+
+static int
+glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+        glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
+        char                        *op_errstr = NULL;
+        glusterd_op_t               op = GD_OP_NONE;
+        gd_node_type                type = GD_NODE_NONE;
+        dict_t                      *op_ctx = NULL;
+        glusterd_req_ctx_t          *req_ctx = NULL;
+        void                        *pending_entry = NULL;
+        xlator_t                    *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (event);
+        GF_ASSERT (ctx);
+        ev_ctx = ctx;
+
+        req_ctx = ev_ctx->commit_ctx;
+        GF_ASSERT (req_ctx);
+
+        op = req_ctx->op;
+        op_ctx = glusterd_op_get_ctx ();
+        pending_entry = ev_ctx->pending_node->node;
+        type = ev_ctx->pending_node->type;
+
+        ret = glusterd_remove_pending_entry (&opinfo.pending_bricks,
+                                             pending_entry);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unknown response received ");
+                ret = -1;
+                goto out;
+        }
+
+        if (opinfo.brick_pending_count > 0)
+                opinfo.brick_pending_count--;
+
+        glusterd_handle_node_rsp (req_ctx->dict, pending_entry, op, ev_ctx->rsp_dict,
+                                  op_ctx, &op_errstr, type);
+
+        if (opinfo.brick_pending_count > 0)
+                goto out;
+
+        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                           ev_ctx->commit_ctx);
+
+out:
+        if (ev_ctx->rsp_dict)
+                dict_unref (ev_ctx->rsp_dict);
+        GF_FREE (ev_ctx);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           struct list_head *selected, dict_t *rsp_dict)
+{
+        int     ret = 0;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (op > GD_OP_NONE);
+        GF_ASSERT (op < GD_OP_MAX);
+
+        switch (op) {
+        case GD_OP_STOP_VOLUME:
+                ret = glusterd_bricks_select_stop_volume (dict, op_errstr,
+                                                          selected);
+                break;
+
+        case GD_OP_REMOVE_BRICK:
+                ret = glusterd_bricks_select_remove_brick (dict, op_errstr,
+                                                           selected);
+                break;
+
+        case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_bricks_select_profile_volume (dict, op_errstr,
+                                                             selected);
+                break;
+
+        case GD_OP_HEAL_VOLUME:
+                ret = glusterd_bricks_select_heal_volume (dict, op_errstr,
+                                                          selected, rsp_dict);
+                break;
+
+        case GD_OP_STATUS_VOLUME:
+                ret = glusterd_bricks_select_status_volume (dict, op_errstr,
+                                                            selected);
+                break;
+
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = glusterd_bricks_select_rebalance_volume (dict, op_errstr,
+                                                               selected);
+                break;
+        case GD_OP_SNAP:
+                ret = glusterd_bricks_select_snap (dict, op_errstr, selected);
+                break;
+        default:
+                break;
+         }
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+glusterd_op_sm_t glusterd_op_state_default [] = {
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_send_lock},//EVENT_START_LOCK
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_ALL_ACK
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_lock_sent [] = {
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_rcvd_lock_acc}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_send_stage_op}, //EVENT_ALL_ACC
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_send_unlock_drain}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_locked [] = {
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_STAGED, glusterd_op_ac_stage_op}, //EVENT_STAGE_OP
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_ALL_ACK
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_stage_op_sent [] = {
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_rcvd_stage_op_acc}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_send_brick_op}, //EVENT_ALL_ACC
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_send_brick_op}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_STAGE_OP_FAILED,   glusterd_op_ac_stage_op_failed}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_stage_op_failed [] = {
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_stage_op_failed}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_stage_op_failed}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_staged [] = {
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_STAGED, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_send_brick_op}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_ALL_ACK
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_brick_op_sent [] = {
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_rcvd_brick_op_acc}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_BRICK_OP_FAILED,   glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_BRICK_OP
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_send_commit_op}, //EVENT_ALL_ACK
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_brick_op_failed [] = {
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_BRICK_OP
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_brick_committed [] = {
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_rcvd_brick_op_acc}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_commit_op}, //EVENT_ALL_ACK
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_brick_commit_failed [] = {
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_send_commit_failed}, //EVENT_ALL_ACK
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_commit_op_failed [] = {
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_commit_op_failed}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_commit_op_failed}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_commit_op_sent [] = {
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_rcvd_commit_op_acc}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_UNLOCK_SENT,    glusterd_op_ac_send_unlock}, //EVENT_ALL_ACC
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_UNLOCK_SENT,    glusterd_op_ac_send_unlock}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_commit_op_failed}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT,        glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_committed [] = {
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_ALL_ACK
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_unlock_sent [] = {
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_rcvd_unlock_acc}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlocked_all}, //EVENT_ALL_ACC
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_rcvd_unlock_acc}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t glusterd_op_state_ack_drain [] = {
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_NONE
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},//EVENT_START_LOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_lock}, //EVENT_LOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_send_unlock_drain}, //EVENT_RCVD_ACC
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_ALL_ACC
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_STAGE_ACC
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_send_unlock_drain}, //EVENT_RCVD_RJT
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_STAGE_OP
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_COMMIT_OP
+        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
+        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
+        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_MAX
+};
+
+glusterd_op_sm_t *glusterd_op_state_table [] = {
+        glusterd_op_state_default,
+        glusterd_op_state_lock_sent,
+        glusterd_op_state_locked,
+        glusterd_op_state_stage_op_sent,
+        glusterd_op_state_staged,
+        glusterd_op_state_commit_op_sent,
+        glusterd_op_state_committed,
+        glusterd_op_state_unlock_sent,
+        glusterd_op_state_stage_op_failed,
+        glusterd_op_state_commit_op_failed,
+        glusterd_op_state_brick_op_sent,
+        glusterd_op_state_brick_op_failed,
+        glusterd_op_state_brick_committed,
+        glusterd_op_state_brick_commit_failed,
+        glusterd_op_state_ack_drain
+};
+
+int
+glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
+                          glusterd_op_sm_event_t **new_event)
+{
+        glusterd_op_sm_event_t      *event = NULL;
+
+        GF_ASSERT (new_event);
+        GF_ASSERT (GD_OP_EVENT_NONE <= event_type &&
+                        GD_OP_EVENT_MAX > event_type);
+
+        event = GF_CALLOC (1, sizeof (*event), gf_gld_mt_op_sm_event_t);
+
+        if (!event)
+                return -1;
+
+        *new_event = event;
+        event->event = event_type;
+        INIT_LIST_HEAD (&event->list);
+
+        return 0;
+}
+
+int
+glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
+                             uuid_t *txn_id, void *ctx)
+{
+        int32_t                 ret = -1;
+        glusterd_op_sm_event_t  *event = NULL;
+
+        GF_ASSERT (event_type < GD_OP_EVENT_MAX &&
+                        event_type >= GD_OP_EVENT_NONE);
+
+        ret = glusterd_op_sm_new_event (event_type, &event);
+
+        if (ret)
+                goto out;
+
+        event->ctx = ctx;
+
+        if (txn_id)
+                uuid_copy (event->txn_id, *txn_id);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Enqueue event: '%s'",
+                glusterd_op_sm_event_name_get (event->event));
+        list_add_tail (&event->list, &gd_op_sm_queue);
+
+out:
+        return ret;
+}
+
+void
+glusterd_destroy_req_ctx (glusterd_req_ctx_t *ctx)
+{
+        if (!ctx)
+                return;
+        if (ctx->dict)
+                dict_unref (ctx->dict);
+        GF_FREE (ctx);
+}
+
+void
+glusterd_destroy_local_unlock_ctx (uuid_t *ctx)
+{
+        if (!ctx)
+                return;
+        GF_FREE (ctx);
+}
+
+void
+glusterd_destroy_op_event_ctx (glusterd_op_sm_event_t *event)
+{
+        if (!event)
+                return;
+
+        switch (event->event) {
+        case GD_OP_EVENT_LOCK:
+        case GD_OP_EVENT_UNLOCK:
+                glusterd_destroy_lock_ctx (event->ctx);
+                break;
+        case GD_OP_EVENT_STAGE_OP:
+        case GD_OP_EVENT_ALL_ACK:
+                glusterd_destroy_req_ctx (event->ctx);
+                break;
+        case GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP:
+                glusterd_destroy_local_unlock_ctx (event->ctx);
+                break;
+        default:
+                break;
+        }
+}
+
+int
+glusterd_op_sm ()
+{
+        glusterd_op_sm_event_t          *event = NULL;
+        glusterd_op_sm_event_t          *tmp = NULL;
+        int                             ret = -1;
+        int                             lock_err = 0;
+        glusterd_op_sm_ac_fn            handler = NULL;
+        glusterd_op_sm_t                *state = NULL;
+        glusterd_op_sm_event_type_t     event_type = GD_OP_EVENT_NONE;
+        xlator_t                        *this = NULL;
+        glusterd_op_info_t              txn_op_info;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if ((lock_err = pthread_mutex_trylock (&gd_op_sm_lock))) {
+                gf_log (this->name, GF_LOG_ERROR, "lock failed due to %s",
+                        strerror (lock_err));
+                goto lock_failed;
+        }
+
+        while (!list_empty (&gd_op_sm_queue)) {
+
+                list_for_each_entry_safe (event, tmp, &gd_op_sm_queue, list) {
+
+                        list_del_init (&event->list);
+                        event_type = event->event;
+                        gf_log (this->name, GF_LOG_DEBUG, "Dequeued event of "
+                                "type: '%s'",
+                                glusterd_op_sm_event_name_get(event_type));
+
+                        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s",
+                                uuid_utoa (event->txn_id));
+
+                        ret = glusterd_get_txn_opinfo (&event->txn_id,
+                                                       &txn_op_info);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to get transaction's opinfo");
+                                glusterd_destroy_op_event_ctx (event);
+                                GF_FREE (event);
+                                continue;
+                        } else
+                                opinfo = txn_op_info;
+
+                        state = glusterd_op_state_table[opinfo.state.state];
+
+                        GF_ASSERT (state);
+
+                        handler = state[event_type].handler;
+                        GF_ASSERT (handler);
+
+                        ret = handler (event, event->ctx);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "handler returned: %d", ret);
+                                glusterd_destroy_op_event_ctx (event);
+                                GF_FREE (event);
+                                continue;
+                        }
+
+                        ret = glusterd_op_sm_transition_state (&opinfo, state,
+                                                                event_type);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to transition"
+                                        "state from '%s' to '%s'",
+                         glusterd_op_sm_state_name_get(opinfo.state.state),
+                         glusterd_op_sm_state_name_get(state[event_type].next_state));
+                                (void ) pthread_mutex_unlock (&gd_op_sm_lock);
+                                return ret;
+                        }
+
+                        if ((state[event_type].next_state ==
+                             GD_OP_STATE_DEFAULT) &&
+                           (event_type == GD_OP_EVENT_UNLOCK)) {
+                                /* Clearing the transaction opinfo */
+                                ret = glusterd_clear_txn_opinfo(&event->txn_id);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Unable to clear "
+                                                "transaction's opinfo");
+                        } else {
+                                ret = glusterd_set_txn_opinfo (&event->txn_id,
+                                                               &opinfo);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Unable to set "
+                                                "transaction's opinfo");
+                        }
+
+                        glusterd_destroy_op_event_ctx (event);
+                        GF_FREE (event);
+
+                }
+        }
+
+
+        (void ) pthread_mutex_unlock (&gd_op_sm_lock);
+        ret = 0;
+
+lock_failed:
+
+        return ret;
+}
+
+int32_t
+glusterd_op_set_op (glusterd_op_t op)
+{
+
+        GF_ASSERT (op < GD_OP_MAX);
+        GF_ASSERT (op > GD_OP_NONE);
+
+        opinfo.op = op;
+
+        return 0;
+
+}
+
+int32_t
+glusterd_op_get_op ()
+{
+
+        return opinfo.op;
+
+}
+
+int32_t
+glusterd_op_set_req (rpcsvc_request_t *req)
+{
+
+        GF_ASSERT (req);
+        opinfo.req = req;
+        return 0;
+}
+
+int32_t
+glusterd_op_clear_op (glusterd_op_t op)
+{
+
+        opinfo.op = GD_OP_NONE;
+
+        return 0;
+
+}
+
+int32_t
+glusterd_op_free_ctx (glusterd_op_t op, void *ctx)
+{
+
+        if (ctx) {
+                switch (op) {
+                case GD_OP_CREATE_VOLUME:
+                case GD_OP_DELETE_VOLUME:
+                case GD_OP_STOP_VOLUME:
+                case GD_OP_ADD_BRICK:
+                case GD_OP_REMOVE_BRICK:
+                case GD_OP_REPLACE_BRICK:
+                case GD_OP_LOG_ROTATE:
+                case GD_OP_SYNC_VOLUME:
+                case GD_OP_SET_VOLUME:
+                case GD_OP_START_VOLUME:
+                case GD_OP_RESET_VOLUME:
+                case GD_OP_GSYNC_SET:
+                case GD_OP_QUOTA:
+                case GD_OP_PROFILE_VOLUME:
+                case GD_OP_STATUS_VOLUME:
+                case GD_OP_REBALANCE:
+                case GD_OP_HEAL_VOLUME:
+                case GD_OP_STATEDUMP_VOLUME:
+                case GD_OP_CLEARLOCKS_VOLUME:
+                case GD_OP_DEFRAG_BRICK_VOLUME:
+                        dict_unref (ctx);
+                        break;
+                default:
+                        GF_ASSERT (0);
+                        break;
+                }
+        }
+
+        glusterd_op_reset_ctx ();
+        return 0;
+
+}
+
+void *
+glusterd_op_get_ctx ()
+{
+
+        return opinfo.op_ctx;
+
+}
+
+int
+glusterd_op_sm_init ()
+{
+        INIT_LIST_HEAD (&gd_op_sm_queue);
+        pthread_mutex_init (&gd_op_sm_lock, NULL);
+        return 0;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
new file mode 100644
index 000000000..cf57b78e0
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -0,0 +1,299 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_OP_SM_H_
+#define _GLUSTERD_OP_SM_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#ifndef GSYNC_CONF_TEMPLATE
+#define GSYNC_CONF_TEMPLATE GEOREP"/gsyncd_template.conf"
+#endif
+
+#include <pthread.h>
+#include "uuid.h"
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "call-stub.h"
+#include "fd.h"
+#include "byte-order.h"
+#include "glusterd.h"
+#include "protocol-common.h"
+
+#define GD_VOLUME_NAME_MAX 256
+#define GD_OP_PROTECTED    (0x02)
+#define GD_OP_UNPROTECTED  (0x04)
+
+typedef enum glusterd_op_sm_state_ {
+        GD_OP_STATE_DEFAULT = 0,
+        GD_OP_STATE_LOCK_SENT,
+        GD_OP_STATE_LOCKED,
+        GD_OP_STATE_STAGE_OP_SENT,
+        GD_OP_STATE_STAGED,
+        GD_OP_STATE_COMMIT_OP_SENT,
+        GD_OP_STATE_COMMITED,
+        GD_OP_STATE_UNLOCK_SENT,
+        GD_OP_STATE_STAGE_OP_FAILED,
+        GD_OP_STATE_COMMIT_OP_FAILED,
+        GD_OP_STATE_BRICK_OP_SENT,
+        GD_OP_STATE_BRICK_OP_FAILED,
+        GD_OP_STATE_BRICK_COMMITTED,
+        GD_OP_STATE_BRICK_COMMIT_FAILED,
+        GD_OP_STATE_ACK_DRAIN,
+        GD_OP_STATE_MAX,
+} glusterd_op_sm_state_t;
+
+typedef enum glusterd_op_sm_event_type_ {
+        GD_OP_EVENT_NONE = 0,
+        GD_OP_EVENT_START_LOCK,
+        GD_OP_EVENT_LOCK,
+        GD_OP_EVENT_RCVD_ACC,
+        GD_OP_EVENT_ALL_ACC,
+        GD_OP_EVENT_STAGE_ACC,
+        GD_OP_EVENT_COMMIT_ACC,
+        GD_OP_EVENT_RCVD_RJT,
+        GD_OP_EVENT_STAGE_OP,
+        GD_OP_EVENT_COMMIT_OP,
+        GD_OP_EVENT_UNLOCK,
+        GD_OP_EVENT_START_UNLOCK,
+        GD_OP_EVENT_ALL_ACK,
+        GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP,
+        GD_OP_EVENT_MAX
+} glusterd_op_sm_event_type_t;
+
+
+struct glusterd_op_sm_event_ {
+        struct list_head                list;
+        void                            *ctx;
+        glusterd_op_sm_event_type_t     event;
+        uuid_t                          txn_id;
+};
+
+typedef struct glusterd_op_sm_event_ glusterd_op_sm_event_t;
+
+typedef int (*glusterd_op_sm_ac_fn) (glusterd_op_sm_event_t *, void *);
+
+typedef struct glusterd_op_sm_ {
+        glusterd_op_sm_state_t      next_state;
+        glusterd_op_sm_ac_fn        handler;
+} glusterd_op_sm_t;
+
+typedef struct glusterd_op_sm_state_info_ {
+        glusterd_op_sm_state_t          state;
+        struct timeval                  time;
+} glusterd_op_sm_state_info_t;
+
+struct glusterd_op_info_ {
+        glusterd_op_sm_state_info_t     state;
+        int32_t                         pending_count;
+        int32_t                         brick_pending_count;
+        int32_t                         op_count;
+        glusterd_op_t                   op;
+        struct list_head                op_peers;
+        void                            *op_ctx;
+        rpcsvc_request_t                *req;
+        int32_t                         op_ret;
+        int32_t                         op_errno;
+        char                            *op_errstr;
+        struct  list_head               pending_bricks;
+};
+
+typedef struct glusterd_op_info_ glusterd_op_info_t;
+
+struct glusterd_op_log_filename_ctx_ {
+        char                    volume_name[GD_VOLUME_NAME_MAX];
+        char                    brick[GD_VOLUME_NAME_MAX];
+        char                    path[PATH_MAX];
+};
+typedef struct glusterd_op_log_filename_ctx_ glusterd_op_log_filename_ctx_t;
+
+struct glusterd_op_lock_ctx_ {
+        uuid_t                  uuid;
+        dict_t                 *dict;
+        rpcsvc_request_t        *req;
+};
+
+typedef struct glusterd_op_lock_ctx_ glusterd_op_lock_ctx_t;
+
+struct glusterd_req_ctx_ {
+        rpcsvc_request_t *req;
+	u_char            uuid[16];
+	int               op;
+        dict_t           *dict;
+};
+
+typedef struct glusterd_req_ctx_ glusterd_req_ctx_t;
+
+typedef struct glusterd_op_brick_rsp_ctx_ {
+        int  op_ret;
+        char *op_errstr;
+        dict_t *rsp_dict;
+        glusterd_req_ctx_t *commit_ctx;
+        glusterd_pending_node_t *pending_node;
+} glusterd_op_brick_rsp_ctx_t;
+
+typedef struct glusterd_pr_brick_rsp_conv_t {
+        int count;
+        dict_t *dict;
+} glusterd_pr_brick_rsp_conv_t;
+
+typedef struct glusterd_heal_rsp_conv_ {
+        dict_t *dict;
+        glusterd_volinfo_t *volinfo;
+        xlator_t *this;
+} glusterd_heal_rsp_conv_t;
+
+typedef struct glusterd_status_rsp_conv_ {
+        int count;
+        int brick_index_max;
+        int other_count;
+        dict_t *dict;
+} glusterd_status_rsp_conv_t;
+
+typedef struct glusterd_gsync_status_temp {
+        dict_t *rsp_dict;
+        glusterd_volinfo_t *volinfo;
+        char *node;
+}glusterd_gsync_status_temp_t;
+
+typedef enum cli_cmd_type_ {
+        PER_REPLICA,
+        ALL_REPLICA,
+ } cli_cmd_type;
+
+int
+glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
+                          glusterd_op_sm_event_t **new_event);
+int
+glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
+                             uuid_t *txn_id, void *ctx);
+
+int
+glusterd_op_sm_init ();
+
+int
+glusterd_op_sm ();
+
+int32_t
+glusterd_op_set_ctx (void *ctx);
+
+int32_t
+glusterd_op_set_op (glusterd_op_t op);
+
+int
+glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx);
+
+int32_t
+glusterd_op_stage_validate (glusterd_op_t op, dict_t *req, char **op_errstr,
+                            dict_t *rsp_dict);
+
+int32_t
+glusterd_op_commit_perform (glusterd_op_t op, dict_t *req, char **op_errstr,
+                            dict_t* dict);
+
+int32_t
+glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                       char *err_str, size_t err_len);
+
+int32_t
+glusterd_op_txn_complete ();
+
+void *
+glusterd_op_get_ctx ();
+
+int32_t
+glusterd_op_set_req (rpcsvc_request_t *req);
+
+int32_t
+glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
+                               int32_t op_errno, rpcsvc_request_t *req,
+                               void *ctx, char *op_errstr);
+int32_t
+glusterd_op_get_op ();
+
+int32_t
+glusterd_op_clear_op ();
+
+int32_t
+glusterd_op_free_ctx (glusterd_op_t op, void *ctx);
+
+int
+glusterd_check_option_exists(char *optstring, char **completion);
+
+int
+set_xlator_option (dict_t *dict, char *key, char *value);
+
+void
+glusterd_do_replace_brick (void *data);
+
+char*
+glusterd_op_sm_state_name_get (int state);
+
+char*
+glusterd_op_sm_event_name_get (int event);
+int32_t
+glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           struct list_head *selected, dict_t *rsp_dict);
+int
+glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickinfo,
+                                 gd1_mgmt_brick_op_req **req, dict_t *dict);
+int
+glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+                               dict_t *dict);
+int32_t
+glusterd_handle_brick_rsp (void *pending_entry, glusterd_op_t op,
+                           dict_t *rsp_dict, dict_t *ctx_dict, char **op_errstr,
+                           gd_node_type type);
+
+dict_t*
+glusterd_op_init_commit_rsp_dict (glusterd_op_t op);
+
+void
+glusterd_op_modify_op_ctx (glusterd_op_t op, void *op_ctx);
+
+int32_t
+glusterd_volume_stats_read_perf (char *brick_path, int32_t blk_size,
+                int32_t blk_count, double *throughput, double *time);
+int32_t
+glusterd_volume_stats_write_perf (char *brick_path, int32_t blk_size,
+                int32_t blk_count, double *throughput, double *time);
+gf_boolean_t
+glusterd_is_volume_started (glusterd_volinfo_t  *volinfo);
+int
+glusterd_start_bricks (glusterd_volinfo_t *volinfo);
+gf_boolean_t
+glusterd_are_all_volumes_stopped ();
+int
+glusterd_stop_bricks (glusterd_volinfo_t *volinfo);
+int
+gsync_status (char *master, char *slave, char *conf_path, int *status);
+
+int
+glusterd_check_gsync_running (glusterd_volinfo_t *volinfo, gf_boolean_t *flag);
+
+int
+glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                 dict_t *op_ctx);
+int
+glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg);
+
+int32_t
+glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+
+int32_t
+glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+
+int32_t
+glusterd_clear_txn_opinfo (uuid_t *txn_id);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c
new file mode 100644
index 000000000..a153ca1a9
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c
@@ -0,0 +1,492 @@
+/*
+   Copyright (c) 2010-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "glusterfs.h"
+#include "compat-errno.h"
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+
+#include "portmap-xdr.h"
+#include "xdr-generic.h"
+#include "protocol-common.h"
+#include "rpcsvc.h"
+
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+
+
+int
+pmap_port_isfree (int port)
+{
+        struct sockaddr_in sin;
+        int                sock = -1;
+        int                ret = 0;
+
+        memset (&sin, 0, sizeof (sin));
+        sin.sin_family = PF_INET;
+        sin.sin_port = hton16 (port);
+
+        sock = socket (PF_INET, SOCK_STREAM, 0);
+        if (sock == -1)
+                return -1;
+
+        ret = bind (sock, (struct sockaddr *)&sin, sizeof (sin));
+        close (sock);
+
+        return (ret == 0) ? 1 : 0;
+}
+
+
+static struct pmap_registry *
+pmap_registry_new (xlator_t *this)
+{
+        struct pmap_registry *pmap = NULL;
+        int                   i = 0;
+
+        pmap = CALLOC (sizeof (*pmap), 1);
+        if (!pmap)
+                return NULL;
+
+        for (i = 0; i < 65536; i++) {
+                if (pmap_port_isfree (i))
+                        pmap->ports[i].type = GF_PMAP_PORT_FREE;
+                else
+                        pmap->ports[i].type = GF_PMAP_PORT_FOREIGN;
+        }
+
+        pmap->base_port = pmap->last_alloc =
+                ((glusterd_conf_t *)(this->private))->base_port;
+
+        return pmap;
+}
+
+
+struct pmap_registry *
+pmap_registry_get (xlator_t *this)
+{
+        glusterd_conf_t      *priv = NULL;
+        struct pmap_registry *pmap = NULL;
+
+        priv = this->private;
+
+        pmap = priv->pmap;
+        if (!pmap) {
+                pmap = pmap_registry_new (this);
+                if (!pmap)
+                        return NULL;
+                priv->pmap = pmap;
+        }
+
+        return pmap;
+}
+
+
+static char*
+nextword (char *str)
+{
+        while (*str && !isspace (*str))
+                str++;
+        while (*str && isspace (*str))
+                str++;
+
+        return str;
+}
+
+int
+pmap_registry_search (xlator_t *this, const char *brickname,
+                      gf_pmap_port_type_t type)
+{
+        struct pmap_registry *pmap = NULL;
+        int                   p = 0;
+        char                 *brck = NULL;
+        char                 *nbrck = NULL;
+
+        pmap = pmap_registry_get (this);
+
+        for (p = pmap->base_port; p <= pmap->last_alloc; p++) {
+                if (!pmap->ports[p].brickname || pmap->ports[p].type != type)
+                        continue;
+
+                for (brck = pmap->ports[p].brickname;;) {
+                        nbrck = strtail (brck, brickname);
+                        if (nbrck && (!*nbrck || isspace (*nbrck)))
+                                return p;
+                        brck = nextword (brck);
+                        if (!*brck)
+                                break;
+                }
+        }
+
+        return 0;
+}
+
+int
+pmap_registry_search_by_xprt (xlator_t *this, void *xprt,
+                              gf_pmap_port_type_t type)
+{
+        struct pmap_registry *pmap = NULL;
+        int                   p    = 0;
+        int                   port = 0;
+
+        pmap = pmap_registry_get (this);
+
+        for (p = pmap->base_port; p <= pmap->last_alloc; p++) {
+                if (!pmap->ports[p].xprt)
+                        continue;
+                if (pmap->ports[p].xprt == xprt &&
+                    pmap->ports[p].type == type) {
+                        port = p;
+                        break;
+                }
+        }
+
+        return port;
+}
+
+
+char *
+pmap_registry_search_by_port (xlator_t *this, int port)
+{
+        struct pmap_registry *pmap = NULL;
+        char *brickname = NULL;
+
+        if (port > 65535)
+                goto out;
+
+        pmap = pmap_registry_get (this);
+
+        if (pmap->ports[port].type == GF_PMAP_PORT_BRICKSERVER)
+                brickname = pmap->ports[port].brickname;
+
+out:
+        return brickname;
+}
+
+
+int
+pmap_registry_alloc (xlator_t *this)
+{
+        struct pmap_registry *pmap = NULL;
+        int                   p = 0;
+        int                   port = 0;
+
+        pmap = pmap_registry_get (this);
+
+        for (p = pmap->last_alloc; p < 65535; p++) {
+                if (pmap->ports[p].type != GF_PMAP_PORT_FREE)
+                        continue;
+
+                if (pmap_port_isfree (p)) {
+                        pmap->ports[p].type = GF_PMAP_PORT_LEASED;
+                        port = p;
+                        break;
+                }
+        }
+
+        if (port)
+                pmap->last_alloc = port;
+
+        return port;
+}
+
+int
+pmap_registry_bind (xlator_t *this, int port, const char *brickname,
+                    gf_pmap_port_type_t type, void *xprt)
+{
+        struct pmap_registry *pmap = NULL;
+        int                   p = 0;
+
+        pmap = pmap_registry_get (this);
+
+        if (port > 65535)
+                goto out;
+
+        p = port;
+        pmap->ports[p].type = type;
+        free (pmap->ports[p].brickname);
+        pmap->ports[p].brickname = strdup (brickname);
+        pmap->ports[p].type = type;
+        pmap->ports[p].xprt = xprt;
+
+        gf_log ("pmap", GF_LOG_INFO, "adding brick %s on port %d",
+                brickname, port);
+
+        if (pmap->last_alloc < p)
+                pmap->last_alloc = p;
+out:
+        return 0;
+}
+
+int
+pmap_registry_remove (xlator_t *this, int port, const char *brickname,
+                      gf_pmap_port_type_t type, void *xprt)
+{
+        struct pmap_registry *pmap = NULL;
+        int                   p = 0;
+        glusterd_conf_t      *priv = NULL;
+
+        priv = this->private;
+        pmap = priv->pmap;
+        if (!pmap)
+                goto out;
+
+        if (port) {
+                if (port > 65535)
+                        goto out;
+
+                p = port;
+                goto remove;
+        }
+
+        if (brickname && strchr (brickname, '/')) {
+                p = pmap_registry_search (this, brickname, type);
+                if (p)
+                        goto remove;
+        }
+
+        if (xprt) {
+                p = pmap_registry_search_by_xprt (this, xprt, type);
+                if (p)
+                        goto remove;
+        }
+
+        goto out;
+remove:
+        gf_log ("pmap", GF_LOG_INFO, "removing brick %s on port %d",
+                pmap->ports[p].brickname, p);
+
+        free (pmap->ports[p].brickname);
+
+        pmap->ports[p].brickname = NULL;
+        pmap->ports[p].xprt = NULL;
+
+out:
+        return 0;
+}
+
+int
+__gluster_pmap_portbybrick (rpcsvc_request_t *req)
+{
+        pmap_port_by_brick_req    args = {0,};
+        pmap_port_by_brick_rsp    rsp  = {0,};
+        char                     *brick = NULL;
+        int                       port = 0;
+        int                       ret = -1;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_port_by_brick_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        brick = args.brick;
+
+        port = pmap_registry_search (THIS, brick, GF_PMAP_PORT_BRICKSERVER);
+
+        if (!port)
+                rsp.op_ret = -1;
+
+        rsp.port = port;
+
+fail:
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_pmap_port_by_brick_rsp);
+        free (args.brick);//malloced by xdr
+
+        return 0;
+}
+
+
+int
+gluster_pmap_portbybrick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_portbybrick);
+}
+
+
+int
+__gluster_pmap_brickbyport (rpcsvc_request_t *req)
+{
+        pmap_brick_by_port_req    args = {0,};
+        pmap_brick_by_port_rsp    rsp  = {0,};
+        int                       ret = -1;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_brick_by_port_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        rsp.brick = pmap_registry_search_by_port (THIS, args.port);
+        if (!rsp.brick) {
+                rsp.op_ret = -1;
+                rsp.brick = "";
+        }
+fail:
+
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_pmap_brick_by_port_rsp);
+
+        return 0;
+}
+
+
+int
+gluster_pmap_brickbyport (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_brickbyport);
+}
+
+
+static int
+glusterd_brick_update_signin (glusterd_brickinfo_t *brickinfo,
+                              gf_boolean_t value)
+{
+        brickinfo->signed_in = value;
+
+        return 0;
+}
+
+int
+__gluster_pmap_signup (rpcsvc_request_t *req)
+{
+        pmap_signup_req    args = {0,};
+        pmap_signup_rsp    rsp  = {0,};
+        int                ret = -1;
+
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_signup_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        rsp.op_ret = pmap_registry_bind (THIS, args.port, args.brick,
+                                         GF_PMAP_PORT_BRICKSERVER, req->trans);
+
+fail:
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_pmap_signup_rsp);
+        free (args.brick);//malloced by xdr
+
+        return 0;
+}
+
+int
+gluster_pmap_signup (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_signup);
+}
+
+int
+__gluster_pmap_signin (rpcsvc_request_t *req)
+{
+        pmap_signin_req    args = {0,};
+        pmap_signin_rsp    rsp  = {0,};
+        glusterd_brickinfo_t *brickinfo = NULL;
+        int                ret = -1;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_signin_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        rsp.op_ret = pmap_registry_bind (THIS, args.port, args.brick,
+                                         GF_PMAP_PORT_BRICKSERVER, req->trans);
+
+        ret = glusterd_get_brickinfo (THIS, args.brick, args.port, _gf_true,
+                                      &brickinfo);
+fail:
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_pmap_signin_rsp);
+        free (args.brick);//malloced by xdr
+
+        if (!ret)
+                glusterd_brick_update_signin (brickinfo, _gf_true);
+
+        return 0;
+}
+
+
+int
+gluster_pmap_signin (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_signin);
+}
+
+
+int
+__gluster_pmap_signout (rpcsvc_request_t *req)
+{
+        pmap_signout_req    args = {0,};
+        pmap_signout_rsp    rsp  = {0,};
+        int                 ret = -1;
+        glusterd_brickinfo_t *brickinfo = NULL;
+
+        ret = xdr_to_generic (req->msg[0], &args,
+                              (xdrproc_t)xdr_pmap_signout_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto fail;
+        }
+
+        rsp.op_ret = pmap_registry_remove (THIS, args.port, args.brick,
+                                           GF_PMAP_PORT_BRICKSERVER, req->trans);
+
+        ret = glusterd_get_brickinfo (THIS, args.brick, args.port, _gf_true,
+                                      &brickinfo);
+fail:
+        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                               (xdrproc_t)xdr_pmap_signout_rsp);
+        free (args.brick);//malloced by xdr
+
+        if (!ret)
+                glusterd_brick_update_signin (brickinfo, _gf_false);
+
+        return 0;
+}
+
+int
+gluster_pmap_signout (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __gluster_pmap_signout);
+}
+
+rpcsvc_actor_t gluster_pmap_actors[] = {
+        [GF_PMAP_NULL]        = {"NULL",        GF_PMAP_NULL,        NULL,                     NULL, 0, DRC_NA},
+        [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", GF_PMAP_PORTBYBRICK, gluster_pmap_portbybrick, NULL, 0, DRC_NA},
+        [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", GF_PMAP_BRICKBYPORT, gluster_pmap_brickbyport, NULL, 0, DRC_NA},
+        [GF_PMAP_SIGNIN]      = {"SIGNIN",      GF_PMAP_SIGNIN,      gluster_pmap_signin,      NULL, 0, DRC_NA},
+        [GF_PMAP_SIGNOUT]     = {"SIGNOUT",     GF_PMAP_SIGNOUT,     gluster_pmap_signout,     NULL, 0, DRC_NA},
+        [GF_PMAP_SIGNUP]      = {"SIGNUP",      GF_PMAP_SIGNUP,      gluster_pmap_signup,      NULL, 0, DRC_NA},
+};
+
+
+struct rpcsvc_program gluster_pmap_prog = {
+        .progname  = "Gluster Portmap",
+        .prognum   = GLUSTER_PMAP_PROGRAM,
+        .progver   = GLUSTER_PMAP_VERSION,
+        .actors    = gluster_pmap_actors,
+        .numactors = GF_PMAP_MAXVALUE,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h
new file mode 100644
index 000000000..6336ee998
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h
@@ -0,0 +1,54 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_PMAP_H_
+#define _GLUSTERD_PMAP_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <pthread.h>
+#include "uuid.h"
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "call-stub.h"
+#include "fd.h"
+#include "byte-order.h"
+#include "glusterd.h"
+#include "rpcsvc.h"
+
+
+#define GF_IANA_PRIV_PORTS_START 49152 /* RFC 6335 */
+
+struct pmap_port_status {
+        gf_pmap_port_type_t type;
+        char  *brickname;
+        void  *xprt;
+};
+
+struct pmap_registry {
+        int     base_port;
+        int     last_alloc;
+        struct  pmap_port_status ports[65536];
+};
+
+int pmap_registry_alloc (xlator_t *this);
+int pmap_registry_bind (xlator_t *this, int port, const char *brickname,
+                        gf_pmap_port_type_t type, void *xprt);
+int pmap_registry_remove (xlator_t *this, int port, const char *brickname,
+                          gf_pmap_port_type_t type, void *xprt);
+int pmap_registry_search (xlator_t *this, const char *brickname,
+                          gf_pmap_port_type_t type);
+struct pmap_registry *pmap_registry_get (xlator_t *this);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
new file mode 100644
index 000000000..318267199
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -0,0 +1,839 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "common-utils.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "run.h"
+
+#include <sys/wait.h>
+
+int
+__glusterd_handle_quota (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        dict_t                         *dict = NULL;
+        glusterd_op_t                   cli_op = GD_OP_QUOTA;
+        char                            operation[256] = {0, };
+        char                           *volname = NULL;
+        int32_t                         type = 0;
+        char                            msg[2048] = {0,};
+        xlator_t                       *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                    "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name, "
+                        "while handling quota command");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Unable to get type of command");
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get type of cmd, "
+                        "while handling quota command");
+               goto out;
+        }
+
+        switch (type) {
+        case GF_QUOTA_OPTION_TYPE_ENABLE:
+                strncpy (operation, "enable", sizeof (operation));
+                break;
+
+        case GF_QUOTA_OPTION_TYPE_DISABLE:
+                strncpy (operation, "disable", sizeof (operation));
+                break;
+
+        case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+                strncpy (operation, "limit-usage", sizeof (operation));
+                break;
+
+        case GF_QUOTA_OPTION_TYPE_REMOVE:
+                strncpy (operation, "remove", sizeof (operation));
+                break;
+        }
+        ret = glusterd_op_begin_synctask (req, GD_OP_QUOTA, dict);
+
+out:
+        if (ret) {
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, msg);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_quota (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_quota);
+}
+
+int32_t
+glusterd_check_if_quota_trans_enabled (glusterd_volinfo_t *volinfo)
+{
+        int32_t  ret           = 0;
+        int      flag          = _gf_false;
+
+        flag = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
+        if (flag == -1) {
+                gf_log ("", GF_LOG_ERROR, "failed to get the quota status");
+                ret = -1;
+                goto out;
+        }
+
+        if (flag == _gf_false) {
+                gf_log ("", GF_LOG_ERROR, "first enable the quota translator");
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+/* At the end of the function, the variable found will be set
+ * to true if the path to be removed was present in the limit-list,
+ * else will be false.
+ */
+int32_t
+_glusterd_quota_remove_limits (char **quota_limits, char *path,
+                               gf_boolean_t *found)
+{
+        int      ret      = 0;
+        int      i        = 0;
+        int      size     = 0;
+        int      len      = 0;
+        int      pathlen  = 0;
+        int      skiplen  = 0;
+        int      flag     = 0;
+        char    *limits   = NULL;
+        char    *qlimits  = NULL;
+
+        if (found != NULL)
+                *found = _gf_false;
+
+        if (*quota_limits == NULL)
+                return -1;
+
+        qlimits = *quota_limits;
+
+        pathlen = strlen (path);
+
+        len = strlen (qlimits);
+
+        limits = GF_CALLOC (len + 1, sizeof (char), gf_gld_mt_char);
+        if (!limits)
+                return -1;
+
+        while (i < len) {
+                if (!memcmp ((void *) &qlimits [i], (void *)path, pathlen))
+                        if (qlimits [i + pathlen] == ':') {
+                                flag = 1;
+                                if (found != NULL)
+                                        *found = _gf_true;
+                        }
+
+                while (qlimits [i + size] != ',' &&
+                       qlimits [i + size] != '\0')
+                        size++;
+
+                if (!flag) {
+                        memcpy ((void *) &limits [i], (void *) &qlimits [i], size + 1);
+                } else {
+                        skiplen = size + 1;
+                        size = len - i - size;
+                        memcpy ((void *) &limits [i], (void *) &qlimits [i + skiplen], size);
+                        break;
+                }
+
+                i += size + 1;
+                size = 0;
+        }
+
+        if (!flag) {
+                ret = 1;
+        } else {
+                len = strlen (limits);
+
+                if (len == 0) {
+                        GF_FREE (qlimits);
+
+                        *quota_limits = NULL;
+
+                        goto out;
+                }
+
+                if (limits[len - 1] == ',') {
+                        limits[len - 1] = '\0';
+                        len --;
+                }
+
+                GF_FREE (qlimits);
+
+                qlimits = GF_CALLOC (len + 1, sizeof (char), gf_gld_mt_char);
+
+                if (!qlimits) {
+                        ret = -1;
+                        goto out;
+                }
+
+                memcpy ((void *) qlimits, (void *) limits, len + 1);
+
+                *quota_limits = qlimits;
+
+                ret = 0;
+        }
+
+out:
+        GF_FREE (limits);
+
+        return ret;
+}
+
+int32_t
+glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname)
+{
+        pid_t                      pid;
+        int32_t                    ret              = 0;
+        int                        status           = 0;
+        char                       mountdir[]       = "/tmp/mntXXXXXX";
+        runner_t                   runner           = {0};
+
+        if (mkdtemp (mountdir) == NULL) {
+                gf_log ("glusterd", GF_LOG_DEBUG,
+                        "failed to create a temporary mount directory");
+                ret = -1;
+                goto out;
+        }
+
+        runinit (&runner);
+        runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                         "-s", "localhost",
+                         "--volfile-id", volname,
+                         "-l", DEFAULT_LOG_FILE_DIRECTORY"/quota-crawl.log",
+                         mountdir, NULL);
+
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run_reuse (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret == -1) {
+                runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
+                runner_end (&runner);
+                goto out;
+        }
+        runner_end (&runner);
+
+        if ((pid = fork ()) < 0) {
+                gf_log ("glusterd", GF_LOG_WARNING, "fork from parent failed");
+                ret = -1;
+                goto out;
+        } else if (pid == 0) {//first child
+                /* fork one more to not hold back main process on
+                 * blocking call below
+                 */
+                pid = fork ();
+                if (pid)
+                        _exit (pid > 0 ? EXIT_SUCCESS : EXIT_FAILURE);
+
+                ret = chdir (mountdir);
+                if (ret == -1) {
+                        gf_log ("glusterd", GF_LOG_WARNING, "chdir %s failed, "
+                                "reason: %s", mountdir, strerror (errno));
+                        exit (EXIT_FAILURE);
+                }
+                runinit (&runner);
+                runner_add_args (&runner, "/usr/bin/find", "find", ".", NULL);
+                if (runner_start (&runner) == -1)
+                        _exit (EXIT_FAILURE);
+
+#ifndef GF_LINUX_HOST_OS
+                runner_end (&runner); /* blocks in waitpid */
+                runcmd ("umount", mountdir, NULL);
+#else
+                runcmd ("umount", "-l", mountdir, NULL);
+#endif
+                rmdir (mountdir);
+                _exit (EXIT_SUCCESS);
+        }
+        ret = (waitpid (pid, &status, 0) == pid &&
+               WIFEXITED (status) && WEXITSTATUS (status) == EXIT_SUCCESS) ? 0 : -1;
+
+out:
+        return ret;
+}
+
+char *
+glusterd_quota_get_limit_value (char *quota_limits, char *path)
+{
+        int32_t i, j, k, l, len;
+        int32_t pat_len, diff;
+        char   *ret_str = NULL;
+
+        len = strlen (quota_limits);
+        pat_len = strlen (path);
+        i = 0;
+        j = 0;
+
+        while (i < len) {
+                j = i;
+                k = 0;
+                while (path [k] == quota_limits [j]) {
+                        j++;
+                        k++;
+                }
+
+                l = j;
+
+                while (quota_limits [j] != ',' &&
+                       quota_limits [j] != '\0')
+                        j++;
+
+                if (quota_limits [l] == ':' && pat_len == (l - i)) {
+                        diff = j - i;
+                        ret_str = GF_CALLOC (diff + 1, sizeof (char),
+                                             gf_gld_mt_char);
+
+                        strncpy (ret_str, &quota_limits [i], diff);
+
+                        break;
+                }
+                i = ++j; //skip ','
+        }
+
+        return ret_str;
+}
+
+char*
+_glusterd_quota_get_limit_usages (glusterd_volinfo_t *volinfo,
+                                  char *path, char **op_errstr)
+{
+        int32_t  ret          = 0;
+        char    *quota_limits = NULL;
+        char    *ret_str      = NULL;
+
+        if (volinfo == NULL)
+                return NULL;
+
+        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_LIMIT_USAGE,
+                                    &quota_limits);
+        if (ret)
+                return NULL;
+        if (quota_limits == NULL) {
+                ret_str = NULL;
+                *op_errstr = gf_strdup ("Limit not set on any directory");
+        } else if (path == NULL)
+                ret_str = gf_strdup (quota_limits);
+        else
+                ret_str = glusterd_quota_get_limit_value (quota_limits, path);
+
+        return ret_str;
+}
+
+int32_t
+glusterd_quota_get_limit_usages (glusterd_conf_t *priv,
+                                 glusterd_volinfo_t *volinfo,
+                                 char *volname,
+                                 dict_t *dict,
+                                 char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        int32_t i               = 0;
+        int32_t ret             = 0;
+        int32_t count           = 0;
+        char    *path           = NULL;
+        char    cmd_str [1024]  = {0, };
+        char   *ret_str         = NULL;
+
+        if (rsp_dict == NULL)
+                return 0;
+
+        ret = dict_get_int32 (dict, "count", &count);
+        if (ret < 0)
+                goto out;
+
+        if (count == 0) {
+                ret_str = _glusterd_quota_get_limit_usages (volinfo, NULL,
+                                                            op_errstr);
+        } else {
+                i = 0;
+                while (count--) {
+                        snprintf (cmd_str, 1024, "path%d", i++);
+
+                        ret = dict_get_str (dict, cmd_str, &path);
+                        if (ret < 0)
+                                goto out;
+
+                        ret_str = _glusterd_quota_get_limit_usages (volinfo, path, op_errstr);
+                }
+        }
+
+        if (ret_str) {
+                ret = dict_set_dynstr (rsp_dict, "limit_list", ret_str);
+        }
+out:
+        return ret;
+}
+
+int32_t
+glusterd_quota_enable (glusterd_volinfo_t *volinfo, char **op_errstr,
+                       gf_boolean_t *crawl)
+{
+        int32_t         ret     = -1;
+        char            *quota_status = NULL;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", crawl, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", op_errstr, out);
+
+        if (glusterd_is_volume_started (volinfo) == 0) {
+                *op_errstr = gf_strdup ("Volume is stopped, start volume "
+                                        "to enable quota.");
+                goto out;
+        }
+
+        ret = glusterd_check_if_quota_trans_enabled (volinfo);
+        if (ret == 0) {
+                *op_errstr = gf_strdup ("Quota is already enabled");
+                goto out;
+        }
+
+        quota_status = gf_strdup ("on");
+        if (!quota_status) {
+                gf_log ("", GF_LOG_ERROR, "memory allocation failed");
+                *op_errstr = gf_strdup ("Enabling quota has been unsuccessful");
+                goto out;
+        }
+
+        ret = dict_set_dynstr (volinfo->dict, VKEY_FEATURES_QUOTA, quota_status);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "dict set failed");
+                *op_errstr = gf_strdup ("Enabling quota has been unsuccessful");
+                goto out;
+        }
+
+        *op_errstr = gf_strdup ("Enabling quota has been successful");
+
+        *crawl = _gf_true;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
+{
+        int32_t  ret            = -1;
+        char    *quota_status   = NULL, *quota_limits = NULL;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", op_errstr, out);
+
+        ret = glusterd_check_if_quota_trans_enabled (volinfo);
+        if (ret == -1) {
+                *op_errstr = gf_strdup ("Quota is already disabled");
+                goto out;
+        }
+
+        quota_status = gf_strdup ("off");
+        if (!quota_status) {
+                gf_log ("", GF_LOG_ERROR, "memory allocation failed");
+                *op_errstr = gf_strdup ("Disabling quota has been unsuccessful");
+                goto out;
+        }
+
+        ret = dict_set_dynstr (volinfo->dict, VKEY_FEATURES_QUOTA, quota_status);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "dict set failed");
+                *op_errstr = gf_strdup ("Disabling quota has been unsuccessful");
+                goto out;
+        }
+
+        *op_errstr = gf_strdup ("Disabling quota has been successful");
+
+        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_LIMIT_USAGE,
+                                    &quota_limits);
+        if (ret) {
+                gf_log ("", GF_LOG_WARNING, "failed to get the quota limits");
+        } else {
+                GF_FREE (quota_limits);
+        }
+
+        dict_del (volinfo->dict, VKEY_FEATURES_LIMIT_USAGE);
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_quota_limit_usage (glusterd_volinfo_t *volinfo, dict_t *dict, char **op_errstr)
+{
+        int32_t          ret    = -1;
+        char            *path   = NULL;
+        char            *limit  = NULL;
+        char            *value  = NULL;
+        char             msg [1024] = {0,};
+        char            *quota_limits = NULL;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", dict, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", op_errstr, out);
+
+        ret = glusterd_check_if_quota_trans_enabled (volinfo);
+        if (ret == -1) {
+                *op_errstr = gf_strdup ("Quota is disabled, please enable "
+                                        "quota");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_LIMIT_USAGE,
+                                    &quota_limits);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "failed to get the quota limits");
+                *op_errstr = gf_strdup ("failed to set limit");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "path", &path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch quota limits" );
+                *op_errstr = gf_strdup ("failed to set limit");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "limit", &limit);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch quota limits" );
+                *op_errstr = gf_strdup ("failed to set limit");
+                goto out;
+        }
+
+        if (quota_limits) {
+                ret = _glusterd_quota_remove_limits (&quota_limits, path, NULL);
+                if (ret == -1) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                        *op_errstr = gf_strdup ("failed to set limit");
+                        goto out;
+                }
+        }
+
+        if (quota_limits == NULL) {
+                ret = gf_asprintf (&value, "%s:%s", path, limit);
+                if (ret == -1) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                        *op_errstr = gf_strdup ("failed to set limit");
+                        goto out;
+                }
+        } else {
+                ret = gf_asprintf (&value, "%s,%s:%s",
+                                   quota_limits, path, limit);
+                if (ret == -1) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                        *op_errstr = gf_strdup ("failed to set limit");
+                        goto out;
+                }
+
+                GF_FREE (quota_limits);
+        }
+
+        quota_limits = value;
+
+        ret = dict_set_str (volinfo->dict, VKEY_FEATURES_LIMIT_USAGE,
+                            quota_limits);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to set quota limits" );
+                *op_errstr = gf_strdup ("failed to set limit");
+                goto out;
+        }
+        snprintf (msg, 1024, "limit set on %s", path);
+        *op_errstr = gf_strdup (msg);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_quota_remove_limits (glusterd_volinfo_t *volinfo, dict_t *dict, char **op_errstr)
+{
+        int32_t         ret                   = -1;
+        char            str [PATH_MAX + 1024] = {0,};
+        char            *quota_limits         = NULL;
+        char            *path                 = NULL;
+        gf_boolean_t     flag                 = _gf_false;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", dict, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", op_errstr, out);
+
+        ret = glusterd_check_if_quota_trans_enabled (volinfo);
+        if (ret == -1) {
+                *op_errstr = gf_strdup ("Quota is disabled, please enable quota");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_LIMIT_USAGE,
+                                    &quota_limits);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "failed to get the quota limits");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "path", &path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to fetch quota limits" );
+                goto out;
+        }
+
+        ret = _glusterd_quota_remove_limits (&quota_limits, path, &flag);
+        if (ret == -1) {
+                if (flag == _gf_true)
+                        snprintf (str, sizeof (str), "Removing limit on %s has "
+                                  "been unsuccessful", path);
+                else
+                        snprintf (str, sizeof (str), "%s has no limit set", path);
+                *op_errstr = gf_strdup (str);
+                goto out;
+        } else {
+                if (flag == _gf_true)
+                        snprintf (str, sizeof (str), "Removed quota limit on "
+                                  "%s", path);
+                else
+                        snprintf (str, sizeof (str), "no limit set on %s",
+                                  path);
+                *op_errstr = gf_strdup (str);
+        }
+
+        if (quota_limits) {
+                ret = dict_set_str (volinfo->dict, VKEY_FEATURES_LIMIT_USAGE,
+                                    quota_limits);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to set quota limits" );
+                        goto out;
+                }
+        } else {
+               dict_del (volinfo->dict, VKEY_FEATURES_LIMIT_USAGE);
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+int
+glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        glusterd_volinfo_t     *volinfo      = NULL;
+        int32_t                 ret          = -1;
+        char                   *volname      = NULL;
+        int                     type         = -1;
+        gf_boolean_t            start_crawl  = _gf_false;
+        glusterd_conf_t        *priv         = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        priv = THIS->private;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name " );
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "type", &type);
+
+        if (type == GF_QUOTA_OPTION_TYPE_ENABLE) {
+                ret = glusterd_quota_enable (volinfo, op_errstr, &start_crawl);
+                if (ret < 0)
+                        goto out;
+
+                goto create_vol;
+        }
+
+        if (type == GF_QUOTA_OPTION_TYPE_DISABLE) {
+                ret = glusterd_quota_disable (volinfo, op_errstr);
+                if (ret < 0)
+                        goto out;
+
+                goto create_vol;
+        }
+
+        if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
+                ret = glusterd_quota_limit_usage (volinfo, dict, op_errstr);
+                if (ret < 0)
+                        goto out;
+
+                goto create_vol;
+        }
+
+        if (type == GF_QUOTA_OPTION_TYPE_REMOVE) {
+                ret = glusterd_quota_remove_limits (volinfo, dict, op_errstr);
+                if (ret < 0)
+                        goto out;
+
+                goto create_vol;
+        }
+
+        if (type == GF_QUOTA_OPTION_TYPE_LIST) {
+                ret = glusterd_check_if_quota_trans_enabled (volinfo);
+                if (ret == -1) {
+                        *op_errstr = gf_strdup ("cannot list the limits, "
+                                                "quota is disabled");
+                        goto out;
+                }
+
+                ret = glusterd_quota_get_limit_usages (priv, volinfo, volname,
+                                                       dict, op_errstr, rsp_dict);
+
+                goto out;
+        }
+create_vol:
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to re-create volfile for"
+                                          " 'quota'");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+                goto out;
+
+        if (GLUSTERD_STATUS_STARTED == volinfo->status)
+                ret = glusterd_check_generate_start_nfs ();
+
+        ret = 0;
+
+out:
+        if (rsp_dict && start_crawl == _gf_true)
+                glusterd_quota_initiate_fs_crawl (priv, volname);
+
+        if (rsp_dict && *op_errstr) {
+                ret = dict_set_dynstr (rsp_dict, "errstr", *op_errstr);
+                if (ret) {
+                        GF_FREE (*op_errstr);
+                        gf_log ("", GF_LOG_DEBUG,
+                                "failed to set error message in ctx");
+                }
+                *op_errstr = NULL;
+        }
+
+        return ret;
+}
+
+int
+glusterd_op_stage_quota (dict_t *dict, char **op_errstr)
+{
+        int                ret           = 0;
+        char              *volname       = NULL;
+        gf_boolean_t       exists        = _gf_false;
+        int                type          = 0;
+        dict_t            *ctx           = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        if (!exists) {
+                gf_log ("", GF_LOG_ERROR, "Volume with name: %s "
+                                "does not exist",
+                                volname);
+                *op_errstr = gf_strdup ("Invalid volume name");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get 'type' for quota op");
+                *op_errstr = gf_strdup ("Volume quota failed, internal error "
+                                        ", unable to get type of operation");
+                goto out;
+        }
+
+
+        ctx = glusterd_op_get_ctx();
+        if (ctx && (type == GF_QUOTA_OPTION_TYPE_ENABLE
+                    || type == GF_QUOTA_OPTION_TYPE_LIST)) {
+                /* Fuse mount req. only for enable & list-usage options*/
+                if (!glusterd_is_fuse_available ()) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Unable to open /dev/"
+                                "fuse (%s), quota command failed",
+                                strerror (errno));
+                        *op_errstr = gf_strdup ("Fuse unavailable");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
new file mode 100644
index 000000000..b7b974c68
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -0,0 +1,749 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <inttypes.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/resource.h>
+#include <sys/statvfs.h>
+
+#include "globals.h"
+#include "compat.h"
+#include "protocol-common.h"
+#include "xlator.h"
+#include "logging.h"
+#include "timer.h"
+#include "glusterd-mem-types.h"
+#include "glusterd.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "run.h"
+#include "glusterd-volgen.h"
+
+#include "syscall.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+
+int32_t
+glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe);
+int
+glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
+                                size_t len, glusterd_op_t op)
+{
+        int      ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        /* Check only if operation is not remove-brick */
+        if ((GD_OP_REMOVE_BRICK != op) &&
+            !gd_is_remove_brick_committed (volinfo)) {
+                gf_log (this->name, GF_LOG_DEBUG, "A remove-brick task on "
+                        "volume %s is not yet committed", volinfo->volname);
+                snprintf (op_errstr, len, "A remove-brick task on volume %s is"
+                          " not yet committed. Either commit or stop the "
+                          "remove-brick task.", volinfo->volname);
+                goto out;
+        }
+
+        if (glusterd_is_defrag_on (volinfo)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "rebalance on volume %s already started",
+                        volinfo->volname);
+                snprintf (op_errstr, len, "Rebalance on %s is already started",
+                          volinfo->volname);
+                goto out;
+        }
+
+        if (glusterd_is_rb_started (volinfo) ||
+            glusterd_is_rb_paused (volinfo)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Rebalance failed as replace brick is in progress on volume %s",
+                        volinfo->volname);
+                snprintf (op_errstr, len, "Rebalance failed as replace brick is in progress on "
+                          "volume %s", volinfo->volname);
+                goto out;
+        }
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+int32_t
+__glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
+{
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_defrag_info_t  *defrag  = NULL;
+        int                     ret      = 0;
+        char                    pidfile[PATH_MAX];
+        glusterd_conf_t        *priv    = NULL;
+
+        priv = THIS->private;
+        if (!priv)
+                return 0;
+
+        volinfo = mydata;
+        if (!volinfo)
+                return 0;
+
+        defrag = volinfo->rebal.defrag;
+        if (!defrag)
+                return 0;
+
+        if ((event == RPC_CLNT_DISCONNECT) && defrag->connected)
+                volinfo->rebal.defrag = NULL;
+
+        GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
+
+        switch (event) {
+        case RPC_CLNT_CONNECT:
+        {
+                if (defrag->connected)
+                        return 0;
+
+                LOCK (&defrag->lock);
+                {
+                        defrag->connected = 1;
+                }
+                UNLOCK (&defrag->lock);
+
+               gf_log ("", GF_LOG_DEBUG, "%s got RPC_CLNT_CONNECT",
+                        rpc->conn.trans->name);
+               break;
+        }
+
+        case RPC_CLNT_DISCONNECT:
+        {
+                if (!defrag->connected)
+                        return 0;
+
+                LOCK (&defrag->lock);
+                {
+                        defrag->connected = 0;
+                }
+                UNLOCK (&defrag->lock);
+
+                if (!glusterd_is_service_running (pidfile, NULL)) {
+                        if (volinfo->rebal.defrag_status ==
+                                                GF_DEFRAG_STATUS_STARTED) {
+                                volinfo->rebal.defrag_status =
+                                                   GF_DEFRAG_STATUS_FAILED;
+                        }
+                 }
+
+                glusterd_store_perform_node_state_store (volinfo);
+
+                if (defrag->rpc) {
+                        rpc_clnt_unref (defrag->rpc);
+                        defrag->rpc = NULL;
+                }
+                if (defrag->cbk_fn)
+                        defrag->cbk_fn (volinfo,
+                                        volinfo->rebal.defrag_status);
+
+                GF_FREE (defrag);
+                gf_log ("", GF_LOG_DEBUG, "%s got RPC_CLNT_DISCONNECT",
+                        rpc->conn.trans->name);
+                break;
+        }
+        default:
+                gf_log ("", GF_LOG_TRACE,
+                        "got some other RPC event %d", event);
+                ret = 0;
+                break;
+        }
+
+        return ret;
+}
+
+int32_t
+glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
+                        rpc_clnt_event_t event, void *data)
+{
+        return glusterd_big_locked_notify (rpc, mydata, event,
+                                           data, __glusterd_defrag_notify);
+}
+
+int
+glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
+                              size_t len, int cmd, defrag_cbk_fn_t cbk,
+                              glusterd_op_t op)
+{
+        int                    ret = -1;
+        glusterd_defrag_info_t *defrag =  NULL;
+        runner_t               runner = {0,};
+        glusterd_conf_t        *priv = NULL;
+        char                   defrag_path[PATH_MAX];
+        char                   sockfile[PATH_MAX] = {0,};
+        char                   pidfile[PATH_MAX] = {0,};
+        char                   logfile[PATH_MAX] = {0,};
+        dict_t                 *options = NULL;
+        char                   valgrind_logfile[PATH_MAX] = {0,};
+
+        priv    = THIS->private;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (op_errstr);
+
+        ret = glusterd_defrag_start_validate (volinfo, op_errstr, len, op);
+        if (ret)
+                goto out;
+        if (!volinfo->rebal.defrag)
+                volinfo->rebal.defrag =
+                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
+                                   gf_gld_mt_defrag_info);
+        if (!volinfo->rebal.defrag)
+                goto out;
+
+        defrag = volinfo->rebal.defrag;
+
+        defrag->cmd = cmd;
+
+        volinfo->rebal.defrag_cmd = cmd;
+        volinfo->rebal.op = op;
+
+        LOCK_INIT (&defrag->lock);
+
+        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+
+        glusterd_volinfo_reset_defrag_stats (volinfo);
+        glusterd_store_perform_node_state_store (volinfo);
+
+        GLUSTERD_GET_DEFRAG_DIR (defrag_path, volinfo, priv);
+        ret = mkdir_p (defrag_path, 0777, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create "
+                        "directory %s", defrag_path);
+                goto out;
+        }
+
+        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo, priv);
+        GLUSTERD_GET_DEFRAG_PID_FILE (pidfile, volinfo, priv);
+        snprintf (logfile, PATH_MAX, "%s/%s-rebalance.log",
+                    DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname);
+        runinit (&runner);
+
+        if (priv->valgrind) {
+                snprintf (valgrind_logfile, PATH_MAX,
+                          "%s/valgrind-%s-rebalance.log",
+                          DEFAULT_LOG_FILE_DIRECTORY,
+                          volinfo->volname);
+
+                runner_add_args (&runner, "valgrind", "--leak-check=full",
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
+                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+        }
+
+        runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                         "-s", "localhost", "--volfile-id", volinfo->volname,
+                         "--xlator-option", "*dht.use-readdirp=yes",
+                         "--xlator-option", "*dht.lookup-unhashed=yes",
+                         "--xlator-option", "*dht.assert-no-child-down=yes",
+                         "--xlator-option", "*replicate*.data-self-heal=off",
+                         "--xlator-option",
+                         "*replicate*.metadata-self-heal=off",
+                         "--xlator-option", "*replicate*.entry-self-heal=off",
+                         "--xlator-option", "*replicate*.readdir-failover=off",
+                         "--xlator-option", "*dht.readdir-optimize=on",
+                         NULL);
+        runner_add_arg (&runner, "--xlator-option");
+        runner_argprintf ( &runner, "*dht.rebalance-cmd=%d",cmd);
+        runner_add_arg (&runner, "--xlator-option");
+        runner_argprintf (&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
+        runner_add_arg (&runner, "--socket-file");
+        runner_argprintf (&runner, "%s",sockfile);
+        runner_add_arg (&runner, "--pid-file");
+        runner_argprintf (&runner, "%s",pidfile);
+        runner_add_arg (&runner, "-l");
+        runner_argprintf (&runner, logfile);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
+        ret = runner_run_nowait (&runner);
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_DEBUG, "rebalance command failed");
+                goto out;
+        }
+
+        sleep (5);
+
+        /* Setting frame-timeout to 10mins (600seconds).
+         * Unix domain sockets ensures that the connection is reliable. The
+         * default timeout of 30mins used for unreliable network connections is
+         * too long for unix domain socket connections.
+         */
+        ret = rpc_transport_unix_options_build (&options, sockfile, 600);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unix options build failed");
+                goto out;
+        }
+
+        synclock_unlock (&priv->big_lock);
+        ret = glusterd_rpc_create (&defrag->rpc, options,
+                                   glusterd_defrag_notify, volinfo);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "RPC create failed");
+                goto out;
+        }
+
+        if (cbk)
+                defrag->cbk_fn = cbk;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+int
+glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
+                               glusterd_conf_t *priv, int cmd)
+{
+        dict_t                  *options = NULL;
+        char                     sockfile[PATH_MAX] = {0,};
+        int                      ret = -1;
+        glusterd_defrag_info_t  *defrag =  NULL;
+
+        if (!volinfo->rebal.defrag)
+                volinfo->rebal.defrag =
+                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
+                                   gf_gld_mt_defrag_info);
+
+        if (!volinfo->rebal.defrag)
+                goto out;
+
+        defrag = volinfo->rebal.defrag;
+
+        defrag->cmd = cmd;
+
+        LOCK_INIT (&defrag->lock);
+
+        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo, priv);
+
+        /* Setting frame-timeout to 10mins (600seconds).
+         * Unix domain sockets ensures that the connection is reliable. The
+         * default timeout of 30mins used for unreliable network connections is
+         * too long for unix domain socket connections.
+         */
+        ret = rpc_transport_unix_options_build (&options, sockfile, 600);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unix options build failed");
+                goto out;
+        }
+
+        synclock_unlock (&priv->big_lock);
+        ret = glusterd_rpc_create (&defrag->rpc, options,
+                                   glusterd_defrag_notify, volinfo);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "RPC create failed");
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_rebalance_cmd_validate (int cmd, char *volname,
+                                 glusterd_volinfo_t **volinfo,
+                                 char *op_errstr, size_t len)
+{
+        int ret = -1;
+
+        if (glusterd_volinfo_find(volname, volinfo)) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Received rebalance on invalid"
+                        " volname %s", volname);
+                snprintf (op_errstr, len, "Volume %s does not exist",
+                          volname);
+                goto out;
+        }
+        if ((*volinfo)->brick_count <= (*volinfo)->dist_leaf_count) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Volume %s is not a "
+                "distribute type or contains only 1 brick", volname);
+                snprintf (op_errstr, len, "Volume %s is not a distribute "
+                          "volume or contains only 1 brick.\n"
+                          "Not performing rebalance", volname);
+                goto out;
+        }
+
+        if ((*volinfo)->status != GLUSTERD_STATUS_STARTED) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Received rebalance on stopped"
+                        " volname %s", volname);
+                snprintf (op_errstr, len, "Volume %s needs to "
+                          "be started to perform rebalance", volname);
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+__glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+{
+        int32_t                 ret       = -1;
+        gf_cli_req              cli_req   = {{0,}};
+        glusterd_conf_t        *priv      = NULL;
+        dict_t                 *dict      = NULL;
+        char                   *volname   = NULL;
+        gf_cli_defrag_type      cmd       = 0;
+        char                    msg[2048] = {0,};
+        xlator_t               *this      = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "rebalance-command", (int32_t*)&cmd);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to get command");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = dict_set_static_bin (dict, "node-uuid", MY_UUID, 16);
+        if (ret)
+                goto out;
+
+        if ((cmd == GF_DEFRAG_CMD_STATUS) ||
+              (cmd == GF_DEFRAG_CMD_STOP)) {
+                ret = glusterd_op_begin (req, GD_OP_DEFRAG_BRICK_VOLUME,
+                                         dict, msg, sizeof (msg));
+        } else
+                ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict,
+                                         msg, sizeof (msg));
+
+out:
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        if (ret) {
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
+                ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0,
+                                                     req, dict, msg);
+
+        }
+
+        free (cli_req.dict.dict_val);//malloced by xdr
+
+        return 0;
+}
+
+int
+glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, __glusterd_handle_defrag_volume);
+}
+
+
+int
+glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
+{
+        char                    *volname = NULL;
+        int                     ret = 0;
+        int32_t                 cmd = 0;
+        char                    msg[2048] = {0};
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *task_id_str = NULL;
+        dict_t                  *op_ctx = NULL;
+        xlator_t                *this = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "volname not found");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "rebalance-command", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "cmd not found");
+                goto out;
+        }
+
+        ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
+                                               msg, sizeof (msg));
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "failed to validate");
+                goto out;
+        }
+        switch (cmd) {
+        case GF_DEFRAG_CMD_START:
+        case GF_DEFRAG_CMD_START_LAYOUT_FIX:
+        case GF_DEFRAG_CMD_START_FORCE:
+                if (is_origin_glusterd (dict)) {
+                        op_ctx = glusterd_op_get_ctx ();
+                        if (!op_ctx) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get op_ctx");
+                                goto out;
+                        }
+
+                        ret = glusterd_generate_and_set_task_id
+                                (op_ctx, GF_REBALANCE_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to generate task-id");
+                                goto out;
+                        }
+                } else {
+                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                snprintf (msg, sizeof (msg),
+                                          "Missing rebalance-id");
+                                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
+                                ret = 0;
+                        }
+                }
+                ret = glusterd_defrag_start_validate (volinfo, msg,
+                                                      sizeof (msg),
+                                                      GD_OP_REBALANCE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                        "start validate failed");
+                        goto out;
+                }
+                break;
+        case GF_DEFRAG_CMD_STATUS:
+        case GF_DEFRAG_CMD_STOP:
+                break;
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        if (ret && op_errstr && msg[0])
+                *op_errstr = gf_strdup (msg);
+
+        return ret;
+}
+
+
+int
+glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        char                    *volname   = NULL;
+        int                     ret       = 0;
+        int32_t                 cmd       = 0;
+        char                    msg[2048] = {0};
+        glusterd_volinfo_t      *volinfo   = NULL;
+        glusterd_conf_t         *priv      = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_brickinfo_t    *tmp      = NULL;
+        gf_boolean_t            volfile_update = _gf_false;
+        char                    *task_id_str = NULL;
+        dict_t                  *ctx = NULL;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "volname not given");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "rebalance-command", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "command not given");
+                goto out;
+        }
+
+
+        ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
+                                               msg, sizeof (msg));
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "cmd validate failed");
+                goto out;
+        }
+
+        /* Set task-id, if available, in op_ctx dict for operations other than
+         * start
+         */
+        if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
+                if (!uuid_is_null (volinfo->rebal.rebalance_id)) {
+                        ctx = glusterd_op_get_ctx ();
+                        if (!ctx) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get op_ctx");
+                                ret = -1;
+                                goto out;
+                        }
+
+                        if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
+                                ret = glusterd_copy_uuid_to_dict
+                                        (volinfo->rebal.rebalance_id, ctx,
+                                         GF_REMOVE_BRICK_TID_KEY);
+                        else
+                                ret = glusterd_copy_uuid_to_dict
+                                        (volinfo->rebal.rebalance_id, ctx,
+                                         GF_REBALANCE_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set task-id");
+                                goto out;
+                        }
+                }
+        }
+
+        switch (cmd) {
+        case GF_DEFRAG_CMD_START:
+        case GF_DEFRAG_CMD_START_LAYOUT_FIX:
+        case GF_DEFRAG_CMD_START_FORCE:
+                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Missing rebalance "
+                                "id");
+                        ret = 0;
+                } else {
+                        uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
+                        volinfo->rebal.op = GD_OP_REBALANCE;
+                }
+                ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
+                                                    cmd, NULL, GD_OP_REBALANCE);
+                 break;
+        case GF_DEFRAG_CMD_STOP:
+                /* Clear task-id only on explicitly stopping rebalance.
+                 * Also clear the stored operation, so it doesn't cause trouble
+                 * with future rebalance/remove-brick starts
+                 */
+                uuid_clear (volinfo->rebal.rebalance_id);
+                volinfo->rebal.op = GD_OP_NONE;
+
+                /* Fall back to the old volume file in case of decommission*/
+                list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
+                                          brick_list) {
+                        if (!brickinfo->decommissioned)
+                                continue;
+                        brickinfo->decommissioned = 0;
+                        volfile_update = _gf_true;
+                }
+
+                if (volfile_update == _gf_false) {
+                        ret = 0;
+                        break;
+                }
+
+                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to create volfiles");
+                        goto out;
+                }
+
+                ret = glusterd_store_volinfo (volinfo,
+                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to store volinfo");
+                        goto out;
+                }
+
+                ret = 0;
+                break;
+
+        case GF_DEFRAG_CMD_STATUS:
+                break;
+        default:
+                break;
+        }
+
+out:
+        if (ret && op_errstr && msg[0])
+                *op_errstr = gf_strdup (msg);
+
+        return ret;
+}
+
+int32_t
+glusterd_defrag_event_notify_handle (dict_t *dict)
+{
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *volname = NULL;
+        int32_t                  ret     = -1;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Failed to get volname");
+                return ret;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Failed to get volinfo for %s"
+                        , volname);
+                return ret;
+        }
+
+        ret = glusterd_defrag_volume_status_update (volinfo, dict);
+
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Failed to update status");
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
new file mode 100644
index 000000000..54b830870
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -0,0 +1,2024 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "common-utils.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterfs.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "run.h"
+#include "syscall.h"
+
+#include <signal.h>
+
+#define GLUSTERD_GET_RB_MNTPT(path, len, volinfo)                           \
+        snprintf (path, len,                                                \
+                  DEFAULT_VAR_RUN_DIRECTORY"/%s-"RB_CLIENT_MOUNTPOINT,      \
+                  volinfo->volname);
+
+extern uuid_t global_txn_id;
+
+int
+glusterd_get_replace_op_str (gf1_cli_replace_op op, char *op_str)
+{
+        int     ret = -1;
+
+        if (!op_str)
+                goto out;
+
+        switch (op) {
+                case GF_REPLACE_OP_START:
+                        strcpy (op_str, "start");
+                        break;
+                case GF_REPLACE_OP_COMMIT:
+                        strcpy (op_str, "commit");
+                        break;
+                case GF_REPLACE_OP_PAUSE:
+                        strcpy (op_str, "pause");
+                        break;
+                case GF_REPLACE_OP_ABORT:
+                        strcpy (op_str, "abort");
+                        break;
+                case GF_REPLACE_OP_STATUS:
+                        strcpy (op_str, "status");
+                        break;
+                case GF_REPLACE_OP_COMMIT_FORCE:
+                        strcpy (op_str, "commit-force");
+                        break;
+                default:
+                        strcpy (op_str, "unknown");
+                        break;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+__glusterd_handle_replace_brick (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        dict_t                          *dict = NULL;
+        char                            *src_brick = NULL;
+        char                            *dst_brick = NULL;
+        int32_t                         op = 0;
+        char                            operation[256];
+        glusterd_op_t                   cli_op = GD_OP_REPLACE_BRICK;
+        char                            *volname = NULL;
+        char                            msg[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Received replace brick req");
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (msg, sizeof (msg), "Unable to decode the "
+                                  "command");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Could not get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "operation", &op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dict_get on operation failed");
+                snprintf (msg, sizeof (msg), "Could not get operation");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "src-brick", &src_brick);
+
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to get src brick");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+        gf_log (this->name, GF_LOG_DEBUG,
+                "src brick=%s", src_brick);
+
+        ret = dict_get_str (dict, "dst-brick", &dst_brick);
+
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to get dest brick");
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        (void) glusterd_get_replace_op_str (op, operation);
+        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
+        gf_log (this->name, GF_LOG_INFO, "Received replace brick %s request",
+                operation);
+
+        ret = glusterd_op_begin (req, GD_OP_REPLACE_BRICK, dict,
+                                 msg, sizeof (msg));
+
+out:
+        free (cli_req.dict.dict_val);//malloced by xdr
+
+        glusterd_friend_sm ();
+        glusterd_op_sm ();
+
+        if (ret) {
+                if (msg[0] == '\0')
+                        snprintf (msg, sizeof (msg), "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, msg);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_replace_brick (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_replace_brick);
+}
+
+static int
+glusterd_get_rb_dst_brickinfo (glusterd_volinfo_t *volinfo,
+                               glusterd_brickinfo_t **brickinfo)
+{
+        int32_t                 ret = -1;
+
+        if (!volinfo || !brickinfo)
+                goto out;
+
+        *brickinfo = volinfo->rep_brick.dst_brick;
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        int                                      ret           = 0;
+        int32_t                                  port          = 0;
+        char                                    *src_brick     = NULL;
+        char                                    *dst_brick     = NULL;
+        char                                    *volname       = NULL;
+        int                                      replace_op    = 0;
+        glusterd_volinfo_t                      *volinfo       = NULL;
+        glusterd_brickinfo_t                    *src_brickinfo = NULL;
+        char                                    *host          = NULL;
+        char                                    *path          = NULL;
+        char                                     msg[2048]     = {0};
+        char                                    *dup_dstbrick  = NULL;
+        glusterd_peerinfo_t                     *peerinfo = NULL;
+        glusterd_brickinfo_t                    *dst_brickinfo = NULL;
+        gf_boolean_t                             is_run        = _gf_false;
+        dict_t                                  *ctx           = NULL;
+        glusterd_conf_t                         *priv          = NULL;
+        char                                    *savetok       = NULL;
+        char                                     pidfile[PATH_MAX] = {0};
+        char                                    *task_id_str = NULL;
+        xlator_t                                *this = NULL;
+        gf_boolean_t                            is_force = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "src-brick", &src_brick);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get src brick");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "src brick=%s", src_brick);
+
+        ret = dict_get_str (dict, "dst-brick", &dst_brick);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get dest brick");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "operation", (int32_t *)&replace_op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dict get on replace-brick operation failed");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "volume: %s does not exist",
+                          volname);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+                ret = -1;
+                snprintf (msg, sizeof (msg), "volume: %s is not started",
+                          volname);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        if (!glusterd_store_is_valid_brickpath (volname, dst_brick) ||
+                !glusterd_is_valid_volfpath (volname, dst_brick)) {
+                snprintf (msg, sizeof (msg), "brick path %s is too "
+                          "long.", dst_brick);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_check_gsync_running (volinfo, &is_run);
+        if (ret && (is_run == _gf_false))
+                gf_log (this->name, GF_LOG_WARNING, "Unable to get the status"
+                                " of active "GEOREP" session");
+        if (is_run) {
+                gf_log (this->name, GF_LOG_WARNING, GEOREP" sessions active"
+                        "for the volume %s ", volname);
+                snprintf (msg, sizeof(msg), GEOREP" sessions are active "
+                                "for the volume %s.\nStop "GEOREP " sessions "
+                                "involved in this volume. Use 'volume "GEOREP
+                                " status' command for more info.",
+                                volname);
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        if (glusterd_is_defrag_on(volinfo)) {
+                snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
+                          "progress. Please retry after completion", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        ctx = glusterd_op_get_ctx();
+
+        switch (replace_op) {
+        case GF_REPLACE_OP_START:
+                if (glusterd_is_rb_started (volinfo)) {
+                        snprintf (msg, sizeof (msg), "Replace brick is already "
+                                  "started for volume");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
+                        ret = -1;
+                        goto out;
+                }
+                if (is_origin_glusterd (dict)) {
+                        if (!ctx) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get op_ctx");
+                                goto out;
+                        }
+
+                        ret = glusterd_generate_and_set_task_id
+                                (ctx, GF_REPLACE_BRICK_TID_KEY);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to generate task-id");
+                                goto out;
+                        }
+
+                } else {
+                        ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY,
+                                            &task_id_str);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Missing replace-brick-id");
+                                ret = 0;
+                        }
+                }
+                is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+                break;
+
+        case GF_REPLACE_OP_PAUSE:
+                if (glusterd_is_rb_paused (volinfo)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is "
+                                "already paused for volume ");
+                        ret = -1;
+                        goto out;
+                } else if (!glusterd_is_rb_started(volinfo)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not"
+                                " started for volume ");
+                        ret = -1;
+                        goto out;
+                }
+                break;
+
+        case GF_REPLACE_OP_ABORT:
+                if (!glusterd_is_rb_ongoing (volinfo)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not"
+                                " started or paused for volume ");
+                        ret = -1;
+                        goto out;
+                }
+                break;
+
+        case GF_REPLACE_OP_COMMIT:
+                if (!glusterd_is_rb_ongoing (volinfo)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not "
+                                "started for volume ");
+                        ret = -1;
+                        goto out;
+                }
+                break;
+
+        case GF_REPLACE_OP_COMMIT_FORCE:
+                is_force = _gf_true;
+                break;
+
+        case GF_REPLACE_OP_STATUS:
+
+                if (glusterd_is_rb_ongoing (volinfo) == _gf_false) {
+                        ret = gf_asprintf (op_errstr, "replace-brick not"
+                                           " started on volume %s",
+                                           volinfo->volname);
+                        if (ret < 0) {
+                                *op_errstr = NULL;
+                                goto out;
+                        }
+
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        ret = -1;
+                        goto out;
+                }
+                break;
+
+        default:
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
+                                                      &src_brickinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "brick: %s does not exist in "
+                          "volume: %s", src_brick, volname);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        if (ctx) {
+                if (!glusterd_is_fuse_available ()) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to open /dev/"
+                                "fuse (%s), replace-brick command failed",
+                                strerror (errno));
+                        snprintf (msg, sizeof(msg), "Fuse unavailable\n "
+                                "Replace-brick failed");
+                        *op_errstr = gf_strdup (msg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (gf_is_local_addr (src_brickinfo->hostname)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "I AM THE SOURCE HOST");
+                if (src_brickinfo->port && rsp_dict) {
+                        ret = dict_set_int32 (rsp_dict, "src-brick-port",
+                                              src_brickinfo->port);
+                        if (ret) {
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "Could not set src-brick-port=%d",
+                                        src_brickinfo->port);
+                        }
+                }
+
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, src_brickinfo,
+                                            priv);
+                if ((replace_op != GF_REPLACE_OP_COMMIT_FORCE) &&
+                    !glusterd_is_service_running (pidfile, NULL)) {
+                        snprintf(msg, sizeof(msg), "Source brick %s:%s "
+                                 "is not online.", src_brickinfo->hostname,
+                                 src_brickinfo->path);
+                        *op_errstr = gf_strdup (msg);
+                        ret = -1;
+                        goto out;
+                }
+
+
+        }
+
+        dup_dstbrick = gf_strdup (dst_brick);
+        if (!dup_dstbrick) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Memory allocation failed");
+                goto out;
+        }
+        host = strtok_r (dup_dstbrick, ":", &savetok);
+        path = strtok_r (NULL, ":", &savetok);
+
+        if (!host || !path) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "dst brick %s is not of form <HOSTNAME>:<export-dir>",
+                        dst_brick);
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_brickinfo_new_from_brick (dst_brick, &dst_brickinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_new_brick_validate (dst_brick, dst_brickinfo,
+                                           msg, sizeof (msg));
+        if (ret) {
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                goto out;
+        }
+
+       if (!glusterd_is_rb_ongoing (volinfo) &&
+            (replace_op == GF_REPLACE_OP_START ||
+             replace_op == GF_REPLACE_OP_COMMIT_FORCE)) {
+
+                volinfo->rep_brick.src_brick = src_brickinfo;
+                volinfo->rep_brick.dst_brick = dst_brickinfo;
+        }
+
+        if (glusterd_rb_check_bricks (volinfo, src_brickinfo, dst_brickinfo)) {
+
+                ret = -1;
+                *op_errstr = gf_strdup ("Incorrect source or "
+                                        "destination brick");
+                if (*op_errstr)
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                goto out;
+       }
+
+        if (!glusterd_is_rb_ongoing (volinfo) &&
+            gf_is_local_addr (host)) {
+                ret = glusterd_validate_and_create_brickpath (dst_brickinfo,
+                                                  volinfo->volume_id,
+                                                  op_errstr, is_force);
+                if (ret)
+                        goto out;
+        }
+
+        if (!gf_is_local_addr (host)) {
+                ret = glusterd_friend_find (NULL, host, &peerinfo);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), "%s, is not a friend",
+                                  host);
+                        *op_errstr = gf_strdup (msg);
+                        goto out;
+                }
+
+                if (!peerinfo->connected) {
+                        snprintf (msg, sizeof (msg), "%s, is not connected at "
+                                  "the moment", host);
+                        *op_errstr = gf_strdup (msg);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state) {
+                        snprintf (msg, sizeof (msg), "%s, is not befriended "
+                                  "at the moment", host);
+                        *op_errstr = gf_strdup (msg);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (replace_op == GF_REPLACE_OP_START &&
+            gf_is_local_addr (volinfo->rep_brick.dst_brick->hostname)) {
+                port = pmap_registry_alloc (THIS);
+                if (!port) {
+                        gf_log (THIS->name, GF_LOG_CRITICAL,
+                                "No free ports available");
+                        ret = -1;
+                        goto out;
+                }
+
+                ctx = glusterd_op_get_ctx();
+                ret = dict_set_int32 ((ctx)?ctx:rsp_dict, "dst-brick-port",
+                                      port);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to set dst "
+                                "brick port");
+                        goto out;
+                }
+                volinfo->rep_brick.dst_brick->port = port;
+        }
+
+        ret = 0;
+
+out:
+        GF_FREE (dup_dstbrick);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+rb_set_mntfd (int mntfd)
+{
+        int     ret = -1;
+        dict_t *ctx = NULL;
+
+        ctx = glusterd_op_get_ctx ();
+        if (!ctx) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Failed to get op ctx");
+                goto out;
+        }
+        ret = dict_set_int32 (ctx, "mntfd", mntfd);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_DEBUG, "Failed to set mnt fd "
+                        "in op ctx");
+out:
+        return ret;
+}
+
+static int
+rb_get_mntfd (int *mntfd)
+{
+        int     ret = -1;
+        dict_t *ctx = NULL;
+
+        ctx = glusterd_op_get_ctx ();
+        if (!ctx) {
+                gf_log (THIS->name, GF_LOG_CRITICAL, "Failed to get op ctx");
+                goto out;
+        }
+        ret = dict_get_int32 (ctx, "mntfd", mntfd);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_DEBUG, "Failed to get mnt fd "
+                        "from op ctx");
+out:
+        return ret;
+}
+
+static int
+rb_regenerate_volfiles (glusterd_volinfo_t *volinfo,
+                        glusterd_brickinfo_t *brickinfo,
+                        int32_t pump_needed)
+{
+        dict_t *dict = NULL;
+        int ret = 0;
+
+        dict = volinfo->dict;
+
+        gf_log ("", GF_LOG_DEBUG,
+                "attempting to set pump value=%d", pump_needed);
+
+        ret = dict_set_int32 (dict, "enable-pump", pump_needed);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "could not dict_set enable-pump");
+                goto out;
+        }
+
+        ret = glusterd_create_rb_volfiles (volinfo, brickinfo);
+
+        dict_del (dict, "enable-pump");
+
+out:
+        return ret;
+}
+
+static int
+rb_src_brick_restart (glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *src_brickinfo,
+                      int activate_pump)
+{
+        int                     ret = 0;
+
+        gf_log ("", GF_LOG_DEBUG,
+                "Attempting to kill src");
+
+        ret = glusterd_nfs_server_stop (volinfo);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to stop nfs, ret: %d",
+                        ret);
+        }
+
+        ret = glusterd_volume_stop_glusterfs (volinfo, src_brickinfo,
+                                              _gf_false);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to stop "
+                        "glusterfs, ret: %d", ret);
+                goto out;
+        }
+
+        glusterd_delete_volfile (volinfo, src_brickinfo);
+
+        if (activate_pump) {
+                ret = rb_regenerate_volfiles (volinfo, src_brickinfo, 1);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not regenerate volfiles with pump");
+                        goto out;
+                }
+        } else {
+                ret = rb_regenerate_volfiles (volinfo, src_brickinfo, 0);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not regenerate volfiles without pump");
+                        goto out;
+                }
+
+        }
+
+        sleep (2);
+        ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo,
+                                               _gf_false);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to start "
+                        "glusterfs, ret: %d", ret);
+                goto out;
+        }
+
+out:
+        ret = glusterd_nfs_server_start (volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to start nfs, ret: %d",
+                        ret);
+        }
+        return ret;
+}
+
+static int
+rb_send_xattr_command (glusterd_volinfo_t *volinfo,
+                       glusterd_brickinfo_t *src_brickinfo,
+                       glusterd_brickinfo_t *dst_brickinfo,
+                       const char *xattr_key, const char *value)
+{
+        int             ret   = -1;
+        int             mntfd = -1;
+
+        ret = rb_get_mntfd (&mntfd);
+        if (ret)
+                goto out;
+
+        ret = sys_fsetxattr (mntfd, xattr_key, value, strlen (value) + 1, 0);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_DEBUG, "setxattr on key: "
+                        "%s, reason: %s", xattr_key, strerror (errno));
+
+out:
+        return ret;
+}
+
+static int
+rb_spawn_dst_brick (glusterd_volinfo_t *volinfo,
+                    glusterd_brickinfo_t *brickinfo)
+{
+        glusterd_conf_t    *priv           = NULL;
+        runner_t            runner         = {0,};
+        int                 ret            = -1;
+        int32_t             port           = 0;
+
+        priv = THIS->private;
+
+        port = brickinfo->port;
+        GF_ASSERT (port);
+
+        runinit (&runner);
+        runner_add_arg (&runner, SBIN_DIR"/glusterfs");
+        runner_argprintf (&runner, "-f" "%s/vols/%s/"RB_DSTBRICKVOL_FILENAME,
+                          priv->workdir, volinfo->volname);
+        runner_argprintf (&runner, "-p" "%s/vols/%s/"RB_DSTBRICK_PIDFILE,
+                          priv->workdir, volinfo->volname);
+        runner_add_arg (&runner, "--xlator-option");
+        runner_argprintf (&runner, "src-server.listen-port=%d", port);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
+        ret = runner_run_nowait (&runner);
+        if (ret) {
+                pmap_registry_remove (THIS, 0, brickinfo->path,
+                                      GF_PMAP_PORT_BRICKSERVER, NULL);
+                gf_log ("", GF_LOG_DEBUG,
+                        "Could not start glusterfs");
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_DEBUG,
+                "Successfully started glusterfs: brick=%s:%s",
+                brickinfo->hostname, brickinfo->path);
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
+rb_spawn_glusterfs_client (glusterd_volinfo_t *volinfo,
+                           glusterd_brickinfo_t *brickinfo)
+{
+        xlator_t           *this            = NULL;
+        glusterd_conf_t    *priv            = NULL;
+        runner_t            runner          = {0,};
+        struct stat         buf             = {0,};
+        char                mntpt[PATH_MAX] = {0,};
+        int                 mntfd           = -1;
+        int                 ret             = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
+        runinit (&runner);
+        runner_add_arg (&runner, SBIN_DIR"/glusterfs");
+        runner_argprintf (&runner, "-f" "%s/vols/%s/"RB_CLIENTVOL_FILENAME,
+                          priv->workdir, volinfo->volname);
+        runner_add_arg (&runner, mntpt);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
+        ret = runner_run_reuse (&runner);
+        if (ret) {
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Could not start glusterfs");
+                runner_end (&runner);
+                goto out;
+        } else {
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Successfully started  glusterfs");
+                runner_end (&runner);
+        }
+
+        ret = stat (mntpt, &buf);
+        if (ret) {
+                 gf_log (this->name, GF_LOG_DEBUG, "stat on mount point %s "
+                         "failed", mntpt);
+                 goto out;
+        }
+
+        mntfd = open (mntpt, O_DIRECTORY);
+        if (mntfd == -1)
+                goto out;
+
+        ret = rb_set_mntfd (mntfd);
+        if (ret)
+                goto out;
+
+        runinit (&runner);
+        runner_add_args (&runner, "/bin/umount", "-l", mntpt, NULL);
+        ret = runner_run_reuse (&runner);
+        if (ret) {
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Lazy unmount failed on maintenance client");
+                runner_end (&runner);
+                goto out;
+        } else {
+                runner_log (&runner, this->name, GF_LOG_DEBUG,
+                            "Successfully unmounted  maintenance client");
+                runner_end (&runner);
+        }
+
+
+out:
+
+        return ret;
+}
+
+static const char *client_volfile_str =  "volume mnt-client\n"
+        " type protocol/client\n"
+        " option remote-host %s\n"
+        " option remote-subvolume %s\n"
+        " option remote-port %d\n"
+        " option transport-type %s\n"
+        " option username %s\n"
+        " option password %s\n"
+        "end-volume\n"
+        "volume mnt-wb\n"
+        " type performance/write-behind\n"
+        " subvolumes mnt-client\n"
+        "end-volume\n";
+
+static int
+rb_generate_client_volfile (glusterd_volinfo_t *volinfo,
+                            glusterd_brickinfo_t *src_brickinfo)
+{
+        glusterd_conf_t  *priv                  = NULL;
+        xlator_t         *this                  = NULL;
+        FILE             *file                  = NULL;
+        char              filename[PATH_MAX]    = {0, };
+        int               ret                   = -1;
+        int               fd                    = -1;
+        char             *ttype                 = NULL;
+
+        this = THIS;
+        priv = this->private;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Creating volfile");
+
+        snprintf (filename, PATH_MAX, "%s/vols/%s/%s",
+                  priv->workdir, volinfo->volname,
+                  RB_CLIENTVOL_FILENAME);
+
+        fd = open (filename, O_CREAT | O_RDONLY, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s", strerror (errno));
+                goto out;
+        }
+        close (fd);
+
+        file = fopen (filename, "w+");
+        if (!file) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Open of volfile failed");
+                ret = -1;
+                goto out;
+        }
+
+        GF_ASSERT (src_brickinfo->port);
+
+	ttype = glusterd_get_trans_type_rb (volinfo->transport_type);
+	if (NULL == ttype){
+		ret = -1;
+		goto out;
+	}
+
+        fprintf (file, client_volfile_str, src_brickinfo->hostname,
+                 src_brickinfo->path,
+                 src_brickinfo->port, ttype,
+                 glusterd_auth_get_username (volinfo),
+                 glusterd_auth_get_password (volinfo));
+
+        fclose (file);
+        GF_FREE (ttype);
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static const char *dst_brick_volfile_str = "volume src-posix\n"
+        " type storage/posix\n"
+        " option directory %s\n"
+        " option volume-id %s\n"
+        "end-volume\n"
+        "volume %s\n"
+        " type features/locks\n"
+        " subvolumes src-posix\n"
+        "end-volume\n"
+        "volume src-server\n"
+        " type protocol/server\n"
+        " option auth.login.%s.allow %s\n"
+        " option auth.login.%s.password %s\n"
+        " option auth.addr.%s.allow *\n"
+        " option transport-type %s\n"
+        " subvolumes %s\n"
+        "end-volume\n";
+
+static int
+rb_generate_dst_brick_volfile (glusterd_volinfo_t *volinfo,
+                               glusterd_brickinfo_t *dst_brickinfo)
+{
+        glusterd_conf_t    *priv                = NULL;
+        xlator_t           *this                = NULL;
+        FILE               *file                = NULL;
+        char                filename[PATH_MAX]  = {0, };
+        int                 ret                 = -1;
+        int                 fd                  = -1;
+        char               *trans_type          = NULL;
+
+        this = THIS;
+        priv = this->private;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Creating volfile");
+
+        snprintf (filename, PATH_MAX, "%s/vols/%s/%s",
+                  priv->workdir, volinfo->volname,
+                  RB_DSTBRICKVOL_FILENAME);
+
+        fd = creat (filename, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s", strerror (errno));
+                goto out;
+        }
+        close (fd);
+
+        file = fopen (filename, "w+");
+        if (!file) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Open of volfile failed");
+                ret = -1;
+                goto out;
+        }
+
+	trans_type = glusterd_get_trans_type_rb (volinfo->transport_type);
+	if (NULL == trans_type){
+		ret = -1;
+		goto out;
+	}
+
+        fprintf (file, dst_brick_volfile_str,
+                 dst_brickinfo->path,
+                 uuid_utoa (volinfo->volume_id),
+                 dst_brickinfo->path,
+                 dst_brickinfo->path,
+                 glusterd_auth_get_username (volinfo),
+                 glusterd_auth_get_username (volinfo),
+                 glusterd_auth_get_password (volinfo),
+                 dst_brickinfo->path,
+                 trans_type,
+                 dst_brickinfo->path);
+
+	GF_FREE (trans_type);
+
+        fclose (file);
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+static int
+rb_mountpoint_mkdir (glusterd_volinfo_t *volinfo,
+                     glusterd_brickinfo_t *src_brickinfo)
+{
+        char             mntpt[PATH_MAX]        = {0,};
+        int              ret                    = -1;
+
+        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
+        ret = mkdir (mntpt, 0777);
+        if (ret && (errno != EEXIST)) {
+                gf_log ("", GF_LOG_DEBUG, "mkdir failed, due to %s",
+                        strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
+rb_mountpoint_rmdir (glusterd_volinfo_t *volinfo,
+                     glusterd_brickinfo_t *src_brickinfo)
+{
+        char             mntpt[PATH_MAX] = {0,};
+        int              ret                        = -1;
+
+        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
+        ret = rmdir (mntpt);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "rmdir failed, reason: %s",
+                        strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
+rb_destroy_maintenance_client (glusterd_volinfo_t *volinfo,
+                               glusterd_brickinfo_t *src_brickinfo)
+{
+        xlator_t         *this                        = NULL;
+        glusterd_conf_t  *priv                        = NULL;
+        char              volfile[PATH_MAX]           = {0,};
+        int               ret                         = -1;
+        int               mntfd                       = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        ret = rb_get_mntfd (&mntfd);
+        if (ret)
+                goto out;
+
+        ret = close (mntfd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to close mount "
+                        "point directory");
+                goto out;
+        }
+
+        ret = rb_mountpoint_rmdir (volinfo, src_brickinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "rmdir of mountpoint "
+                        "failed");
+                goto out;
+        }
+
+        snprintf (volfile, PATH_MAX, "%s/vols/%s/%s", priv->workdir,
+                  volinfo->volname, RB_CLIENTVOL_FILENAME);
+
+        ret = unlink (volfile);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "unlink of %s failed, reason: %s",
+                        volfile, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
+rb_spawn_maintenance_client (glusterd_volinfo_t *volinfo,
+                             glusterd_brickinfo_t *src_brickinfo)
+{
+        int ret = -1;
+
+        ret = rb_generate_client_volfile (volinfo, src_brickinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Unable to generate client "
+                        "volfile");
+                goto out;
+        }
+
+        ret = rb_mountpoint_mkdir (volinfo, src_brickinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Unable to mkdir "
+                        "mountpoint");
+                goto out;
+        }
+
+        ret = rb_spawn_glusterfs_client (volinfo, src_brickinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Unable to start glusterfs");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+rb_spawn_destination_brick (glusterd_volinfo_t *volinfo,
+                            glusterd_brickinfo_t *dst_brickinfo)
+
+{
+        int ret = -1;
+
+        ret = rb_generate_dst_brick_volfile (volinfo, dst_brickinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Unable to generate client "
+                        "volfile");
+                goto out;
+        }
+
+        ret = rb_spawn_dst_brick (volinfo, dst_brickinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Unable to start glusterfs");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+rb_kill_destination_brick (glusterd_volinfo_t *volinfo,
+                           glusterd_brickinfo_t *dst_brickinfo)
+{
+        glusterd_conf_t  *priv               = NULL;
+        char              pidfile[PATH_MAX]  = {0,};
+
+        priv = THIS->private;
+
+        snprintf (pidfile, PATH_MAX, "%s/vols/%s/%s",
+                  priv->workdir, volinfo->volname,
+                  RB_DSTBRICK_PIDFILE);
+
+        return glusterd_service_stop ("brick", pidfile, SIGTERM, _gf_true);
+}
+
+static int
+rb_get_xattr_command (glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *src_brickinfo,
+                      glusterd_brickinfo_t *dst_brickinfo,
+                      const char *xattr_key,
+                      char *value)
+{
+        int             ret    = -1;
+        int             mntfd  = -1;
+
+        ret = rb_get_mntfd (&mntfd);
+        if (ret)
+                goto out;
+
+        ret = sys_fgetxattr (mntfd, xattr_key, value, 8192);
+
+        if (ret < 0) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "getxattr on key: %s "
+                        "failed, reason: %s", xattr_key, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+rb_send_cmd (glusterd_volinfo_t *volinfo,
+             glusterd_brickinfo_t *src,
+             glusterd_brickinfo_t *dst,
+             gf1_cli_replace_op op)
+{
+        char         start_value[8192]          = {0,};
+        char         status_str[8192]           = {0,};
+        char        *status_reply               = NULL;
+        char        *tmp                        = NULL;
+        char        *save_ptr                   = NULL;
+        char         filename[PATH_MAX]         = {0,};
+        char        *current_file               = NULL;
+        uint64_t     files                      = 0;
+        int          status                     = 0;
+        dict_t      *ctx                        = NULL;
+        int          ret                        = 0;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (src);
+        GF_ASSERT (dst);
+        GF_ASSERT ((op > GF_REPLACE_OP_NONE)
+                   && (op <= GF_REPLACE_OP_COMMIT_FORCE));
+
+        switch (op) {
+                case GF_REPLACE_OP_START:
+                {
+                        snprintf (start_value, sizeof (start_value),
+                                  "%s:%s:%d", dst->hostname, dst->path,
+                                  dst->port);
+                        ret = rb_send_xattr_command (volinfo, src, dst,
+                                                     RB_PUMP_CMD_START,
+                                                     start_value);
+                }
+                break;
+                case GF_REPLACE_OP_PAUSE:
+                {
+                        ret = rb_send_xattr_command (volinfo, src, dst,
+                                                     RB_PUMP_CMD_PAUSE,
+                                                     RB_PUMP_DEF_ARG);
+                }
+                break;
+                case GF_REPLACE_OP_ABORT:
+                {
+                        ret = rb_send_xattr_command (volinfo, src, dst,
+                                                     RB_PUMP_CMD_ABORT,
+                                                     RB_PUMP_DEF_ARG);
+                }
+                break;
+                case GF_REPLACE_OP_COMMIT:
+                {
+                        ret = rb_send_xattr_command (volinfo, src, dst,
+                                                     RB_PUMP_CMD_COMMIT,
+                                                     RB_PUMP_DEF_ARG);
+                }
+                break;
+                case GF_REPLACE_OP_STATUS:
+                {
+                        ret = rb_get_xattr_command (volinfo, src, dst,
+                                                    RB_PUMP_CMD_STATUS,
+                                                    status_str);
+                        if (ret)
+                                goto out;
+
+                        ctx = glusterd_op_get_ctx ();
+                        GF_ASSERT (ctx);
+                        if (!ctx) {
+                                ret = -1;
+                                gf_log (THIS->name, GF_LOG_CRITICAL,
+                                        "ctx is not present.");
+                                goto out;
+                        }
+
+                        /* Split status reply into different parts */
+                        tmp = strtok_r (status_str, ":", &save_ptr);
+                        if (!tmp) {
+                                ret = -1;
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Couldn't tokenize status string");
+                                goto out;
+                        }
+                        sscanf (tmp, "status=%d", &status);
+                        ret = dict_set_int32 (ctx, "status", status);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
+                                        "set rb status in context");
+                                goto out;
+                        }
+
+                        tmp = NULL;
+                        tmp = strtok_r (NULL, ":", &save_ptr);
+                        if (!tmp) {
+                                ret = -1;
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Couldn't tokenize status string");
+                                goto out;
+                        }
+                        sscanf (tmp, "no_of_files=%"SCNu64, &files);
+                        ret = dict_set_uint64 (ctx, "files", files);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
+                                        "set rb files in context");
+                                goto out;
+                        }
+
+                        if (status == 0) {
+                                tmp = NULL;
+                                tmp = strtok_r (NULL, ":", &save_ptr);
+                                if (!tmp) {
+                                        ret = -1;
+                                        gf_log (THIS->name, GF_LOG_ERROR,
+                                                "Couldn't tokenize status "
+                                                "string");
+                                        goto out;
+                                }
+                                sscanf (tmp, "current_file=%s", filename);
+                                current_file = gf_strdup (filename);
+                                ret = dict_set_dynstr (ctx, "current_file",
+                                                       current_file);
+                                if (ret) {
+                                        GF_FREE (current_file);
+                                        gf_log (THIS->name, GF_LOG_ERROR,
+                                                "Couldn't set rb current file "
+                                                "in context");
+                                        goto out;
+                                }
+                        }
+                        if (status) {
+                                ret = gf_asprintf (&status_reply,
+                                                  "Number of files migrated = %"
+                                                  PRIu64"\tMigration complete",
+                                                  files);
+                        } else {
+                                ret = gf_asprintf (&status_reply,
+                                                  "Number of files migrated = %"
+                                                  PRIu64"\tCurrent file = %s",
+                                                  files, filename);
+                        }
+                        if (ret == -1) {
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Failed to create status_reply string");
+                                goto out;
+                        }
+                        ret = dict_set_dynstr (ctx, "status-reply",
+                                               status_reply);
+                        if (ret) {
+                                GF_FREE (status_reply);
+                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
+                                        "set rb status response in context.");
+                                goto out;
+                        }
+                }
+                break;
+                default:
+                {
+                        GF_ASSERT (0);
+                        ret = -1;
+                        gf_log (THIS->name, GF_LOG_CRITICAL, "Invalid replace"
+                                " brick subcommand.");
+                }
+                break;
+        }
+out:
+        return ret;
+}
+
+static int
+rb_do_operation (glusterd_volinfo_t *volinfo,
+                 glusterd_brickinfo_t *src_brickinfo,
+                 glusterd_brickinfo_t *dst_brickinfo,
+                 gf1_cli_replace_op op)
+{
+
+        int             ret             = -1;
+        char            op_str[256]     = {0, };
+        xlator_t        *this           = NULL;
+
+        this = THIS;
+
+        ret = rb_spawn_maintenance_client (volinfo, src_brickinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Could not spawn "
+                        "maintenance client");
+                goto umount;
+        }
+
+        ret = rb_send_cmd (volinfo, src_brickinfo, dst_brickinfo, op);
+        if (ret) {
+                (void) glusterd_get_replace_op_str (op, op_str);
+                gf_log (this->name, GF_LOG_DEBUG, "Sending replace-brick "
+                        "sub-command %s failed.", op_str);
+        }
+
+umount:
+        if (rb_destroy_maintenance_client (volinfo, src_brickinfo))
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to destroy "
+                        "maintenance client");
+
+         return ret;
+}
+
+/* Set src-brick's port number to be used in the maintenance mount
+ * after all commit acks are received.
+ */
+static int
+rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
+                         dict_t *req_dict, int32_t replace_op)
+{
+        xlator_t *this            = NULL;
+        dict_t   *ctx             = NULL;
+        int       ret             = 0;
+        int       dict_ret        = 0;
+        int       src_port        = 0;
+
+        this = THIS;
+
+        dict_ret = dict_get_int32 (req_dict, "src-brick-port", &src_port);
+        if (src_port)
+                src_brickinfo->port = src_port;
+
+        if (gf_is_local_addr (src_brickinfo->hostname)) {
+                gf_log ("", GF_LOG_INFO,
+                        "adding src-brick port no");
+
+                src_brickinfo->port = pmap_registry_search (this,
+                                      src_brickinfo->path, GF_PMAP_PORT_BRICKSERVER);
+                if (!src_brickinfo->port &&
+                    replace_op != GF_REPLACE_OP_COMMIT_FORCE ) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Src brick port not available");
+                        ret = -1;
+                        goto out;
+                }
+
+                if (rsp_dict) {
+                        ret = dict_set_int32 (rsp_dict, "src-brick-port", src_brickinfo->port);
+                        if (ret) {
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "Could not set src-brick port no");
+                                goto out;
+                        }
+                }
+
+                ctx = glusterd_op_get_ctx ();
+                if (ctx) {
+                        ret = dict_set_int32 (ctx, "src-brick-port", src_brickinfo->port);
+                        if (ret) {
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "Could not set src-brick port no");
+                                goto out;
+                        }
+                }
+
+        }
+
+out:
+        return ret;
+
+}
+
+static int
+rb_update_dstbrick_port (glusterd_brickinfo_t *dst_brickinfo, dict_t *rsp_dict,
+                         dict_t *req_dict, int32_t replace_op)
+{
+        dict_t *ctx           = NULL;
+        int     ret           = 0;
+        int     dict_ret      = 0;
+        int     dst_port      = 0;
+
+        dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port);
+        if (!dict_ret)
+                dst_brickinfo->port = dst_port;
+
+
+        if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                gf_log ("", GF_LOG_INFO,
+                        "adding dst-brick port no");
+
+                if (rsp_dict) {
+                        ret = dict_set_int32 (rsp_dict, "dst-brick-port",
+                                              dst_brickinfo->port);
+                        if (ret) {
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "Could not set dst-brick port no in rsp dict");
+                                goto out;
+                        }
+                }
+
+                ctx = glusterd_op_get_ctx ();
+                if (ctx) {
+                        ret = dict_set_int32 (ctx, "dst-brick-port",
+                                              dst_brickinfo->port);
+                        if (ret) {
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "Could not set dst-brick port no");
+                                goto out;
+                        }
+                }
+        }
+out:
+        return ret;
+}
+
+static int
+glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
+                                   char *old_brick, char  *new_brick)
+{
+        glusterd_brickinfo_t                    *old_brickinfo = NULL;
+        glusterd_brickinfo_t                    *new_brickinfo = NULL;
+        int32_t                                 ret = -1;
+
+        GF_ASSERT (volinfo);
+
+        ret = glusterd_brickinfo_new_from_brick (new_brick,
+                                                 &new_brickinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_resolve_brick (new_brickinfo);
+
+        if (ret)
+                goto out;
+
+        ret = glusterd_volume_brickinfo_get_by_brick (old_brick,
+                                                      volinfo, &old_brickinfo);
+        if (ret)
+                goto out;
+
+        list_add_tail (&new_brickinfo->brick_list,
+                       &old_brickinfo->brick_list);
+
+        volinfo->brick_count++;
+
+        ret = glusterd_op_perform_remove_brick (volinfo, old_brick, 1, NULL);
+        if (ret)
+                goto out;
+
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret)
+                goto out;
+
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                ret = glusterd_brick_start (volinfo, new_brickinfo, _gf_false);
+                if (ret)
+                        goto out;
+        }
+
+out:
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
+{
+        int                                      ret = 0;
+        dict_t                                  *ctx  = NULL;
+        int                                      replace_op = 0;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        char                                    *volname = NULL;
+        xlator_t                                *this = NULL;
+        glusterd_conf_t                         *priv = NULL;
+        char                                    *src_brick = NULL;
+        char                                    *dst_brick = NULL;
+        glusterd_brickinfo_t                    *src_brickinfo = NULL;
+        glusterd_brickinfo_t                    *dst_brickinfo = NULL;
+        char                                    *task_id_str = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "src-brick", &src_brick);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get src brick");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "src brick=%s", src_brick);
+
+        ret = dict_get_str (dict, "dst-brick", &dst_brick);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get dst brick");
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "operation", (int32_t *)&replace_op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dict_get on operation failed");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+        ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
+                                                      &src_brickinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to get src-brickinfo");
+                goto out;
+        }
+
+
+        ret = glusterd_get_rb_dst_brickinfo (volinfo, &dst_brickinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get "
+                         "replace brick destination brickinfo");
+                goto out;
+        }
+
+        ret = glusterd_resolve_brick (dst_brickinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Unable to resolve dst-brickinfo");
+                goto out;
+        }
+
+        ret = rb_update_srcbrick_port (src_brickinfo, rsp_dict,
+                                       dict, replace_op);
+        if (ret)
+                goto out;
+
+
+	if ((GF_REPLACE_OP_START != replace_op)) {
+
+                /* Set task-id, if available, in op_ctx dict for operations
+                 * other than start
+                 */
+                if  (is_origin_glusterd (dict)) {
+                        ctx = glusterd_op_get_ctx();
+                        if (!ctx) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "get op_ctx");
+                                ret = -1;
+                                goto out;
+                        }
+                        if (!uuid_is_null (volinfo->rep_brick.rb_id)) {
+                                ret = glusterd_copy_uuid_to_dict
+                                        (volinfo->rep_brick.rb_id, ctx,
+                                         GF_REPLACE_BRICK_TID_KEY);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to set "
+                                                "replace-brick-id");
+                                        goto out;
+                                }
+                        }
+                }
+	}
+        ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
+                                       dict, replace_op);
+        if (ret)
+                goto out;
+
+        switch (replace_op) {
+        case GF_REPLACE_OP_START:
+        {
+                ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY, &task_id_str);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Missing replace-brick-id");
+                        ret = 0;
+                } else {
+                        uuid_parse (task_id_str, volinfo->rep_brick.rb_id);
+                }
+
+                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "I AM THE DESTINATION HOST");
+                        if (!glusterd_is_rb_paused (volinfo)) {
+                                ret = rb_spawn_destination_brick
+                                        (volinfo, dst_brickinfo);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "Failed to spawn destination "
+                                                "brick");
+                                        goto out;
+                                }
+                        } else {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Replace brick is already started=> no "
+                                        "need to restart dst brick ");
+                        }
+		}
+
+
+		if (gf_is_local_addr (src_brickinfo->hostname)) {
+		        ret = rb_src_brick_restart (volinfo, src_brickinfo,
+				                    1);
+		        if (ret) {
+				gf_log (this->name, GF_LOG_DEBUG,
+	                                "Could not restart src-brick");
+			        goto out;
+			}
+		}
+
+		if (gf_is_local_addr (dst_brickinfo->hostname)) {
+			gf_log (this->name, GF_LOG_INFO,
+				"adding dst-brick port no");
+
+                        ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
+                                                       dict, replace_op);
+                        if (ret)
+                                goto out;
+                }
+
+                glusterd_set_rb_status (volinfo, GF_RB_STATUS_STARTED);
+		break;
+	}
+
+        case GF_REPLACE_OP_COMMIT:
+        {
+                ctx = glusterd_op_get_ctx ();
+                if (ctx) {
+                        ret = rb_do_operation (volinfo, src_brickinfo,
+                                               dst_brickinfo,
+                                               GF_REPLACE_OP_COMMIT);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Commit operation failed");
+                                goto out;
+                        }
+                }
+        }
+                /* fall through */
+        case GF_REPLACE_OP_COMMIT_FORCE:
+        {
+                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "I AM THE DESTINATION HOST");
+                        ret = rb_kill_destination_brick (volinfo,
+                                                         dst_brickinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_CRITICAL,
+                                        "Unable to cleanup dst brick");
+                                goto out;
+                        }
+                }
+
+                ret = glusterd_nodesvcs_stop (volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to stop nfs server, ret: %d", ret);
+                }
+
+		ret = glusterd_op_perform_replace_brick (volinfo, src_brick,
+							 dst_brick);
+		if (ret) {
+			gf_log (this->name, GF_LOG_CRITICAL, "Unable to add "
+				"dst-brick: %s to volume: %s", dst_brick,
+                                volinfo->volname);
+		        (void) glusterd_nodesvcs_handle_graph_change (volinfo);
+			goto out;
+		}
+
+		volinfo->rebal.defrag_status = 0;
+
+		ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+		if (ret) {
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Failed to generate nfs volume file");
+		}
+
+
+		ret = glusterd_fetchspec_notify (THIS);
+                glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE);
+                glusterd_brickinfo_delete (volinfo->rep_brick.dst_brick);
+                volinfo->rep_brick.src_brick = NULL;
+                volinfo->rep_brick.dst_brick = NULL;
+                uuid_clear (volinfo->rep_brick.rb_id);
+        }
+        break;
+
+        case GF_REPLACE_OP_PAUSE:
+        {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Received pause - doing nothing");
+                ctx = glusterd_op_get_ctx ();
+                if (ctx) {
+                        ret = rb_do_operation (volinfo, src_brickinfo,
+                                               dst_brickinfo,
+                                               GF_REPLACE_OP_PAUSE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Pause operation failed");
+                                goto out;
+                        }
+                }
+
+                glusterd_set_rb_status (volinfo, GF_RB_STATUS_PAUSED);
+        }
+                break;
+
+        case GF_REPLACE_OP_ABORT:
+        {
+
+                ctx = glusterd_op_get_ctx ();
+                if (ctx) {
+                        ret = rb_do_operation (volinfo, src_brickinfo,
+                                               dst_brickinfo,
+                                               GF_REPLACE_OP_ABORT);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Abort operation failed");
+                                goto out;
+                        }
+                }
+
+                if (gf_is_local_addr (src_brickinfo->hostname)) {
+                        ret = rb_src_brick_restart (volinfo, src_brickinfo,
+                                                    0);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Couldn't restart src brick "
+                                        "with pump xlator disabled.");
+                                goto out;
+                        }
+                }
+
+                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "I AM THE DESTINATION HOST");
+                        ret = rb_kill_destination_brick (volinfo, dst_brickinfo);
+                        if (ret) {
+                                gf_log ("", GF_LOG_DEBUG,
+                                        "Failed to kill destination brick");
+                                goto out;
+                        }
+                }
+                glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE);
+                glusterd_brickinfo_delete (volinfo->rep_brick.dst_brick);
+                volinfo->rep_brick.src_brick = NULL;
+                volinfo->rep_brick.dst_brick = NULL;
+        }
+        break;
+
+        case GF_REPLACE_OP_STATUS:
+        {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "received status - doing nothing");
+                ctx = glusterd_op_get_ctx ();
+                if (ctx) {
+                        if (glusterd_is_rb_paused (volinfo)) {
+                                ret = dict_set_str (ctx, "status-reply",
+                                                 "replace brick has been paused");
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "failed to set pump status"
+                                                " in ctx");
+                                goto out;
+                        }
+
+                        ret = rb_do_operation (volinfo, src_brickinfo,
+                                               dst_brickinfo,
+                                               GF_REPLACE_OP_STATUS);
+                        if (ret)
+                                goto out;
+                }
+
+        }
+                break;
+
+        default:
+                ret = -1;
+                goto out;
+        }
+        if (!ret && replace_op != GF_REPLACE_OP_STATUS)
+		ret = glusterd_store_volinfo (volinfo,
+                                              GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't store"
+                        " replace brick operation's state");
+
+out:
+        return ret;
+}
+
+void
+glusterd_do_replace_brick (void *data)
+{
+        glusterd_volinfo_t     *volinfo = NULL;
+        int32_t                 op      = 0;
+        int32_t                 src_port = 0;
+        int32_t                 dst_port = 0;
+        dict_t                 *dict    = NULL;
+        char                   *src_brick = NULL;
+        char                   *dst_brick = NULL;
+        char                   *volname   = NULL;
+        glusterd_brickinfo_t   *src_brickinfo = NULL;
+        glusterd_brickinfo_t   *dst_brickinfo = NULL;
+	glusterd_conf_t	       *priv = NULL;
+        uuid_t                 *txn_id = &global_txn_id;
+
+        int ret = 0;
+
+        dict = data;
+
+	GF_ASSERT (THIS);
+
+	priv = THIS->private;
+
+	if (priv->timer) {
+		gf_timer_call_cancel (THIS->ctx, priv->timer);
+		priv->timer = NULL;
+                gf_log ("", GF_LOG_DEBUG,
+                        "Cancelling timer thread");
+	}
+
+        gf_log ("", GF_LOG_DEBUG,
+                "Replace brick operation detected");
+
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        ret = dict_get_int32 (dict, "operation", &op);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "dict_get on operation failed");
+                goto out;
+        }
+        ret = dict_get_str (dict, "src-brick", &src_brick);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get src brick");
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_DEBUG,
+                "src brick=%s", src_brick);
+
+        ret = dict_get_str (dict, "dst-brick", &dst_brick);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get dst brick");
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_DEBUG,
+                "dst brick=%s", dst_brick);
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+        ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
+                                                      &src_brickinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Unable to get src-brickinfo");
+                goto out;
+        }
+
+        ret = glusterd_get_rb_dst_brickinfo (volinfo, &dst_brickinfo);
+        if (!dst_brickinfo) {
+                gf_log ("", GF_LOG_DEBUG, "Unable to get dst-brickinfo");
+                goto out;
+        }
+
+        ret = glusterd_resolve_brick (dst_brickinfo);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Unable to resolve dst-brickinfo");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "src-brick-port", &src_port);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get src-brick port");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "dst-brick-port", &dst_port);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get dst-brick port");
+        }
+
+        dst_brickinfo->port = dst_port;
+        src_brickinfo->port = src_port;
+
+        switch (op) {
+        case GF_REPLACE_OP_START:
+                if (!dst_port) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = rb_do_operation (volinfo, src_brickinfo, dst_brickinfo,
+                                       GF_REPLACE_OP_START);
+                if (ret)
+                        goto out;
+                break;
+        case GF_REPLACE_OP_PAUSE:
+        case GF_REPLACE_OP_ABORT:
+        case GF_REPLACE_OP_COMMIT:
+        case GF_REPLACE_OP_COMMIT_FORCE:
+        case GF_REPLACE_OP_STATUS:
+                break;
+        default:
+                ret = -1;
+                goto out;
+        }
+
+out:
+        if (ret)
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                                   txn_id, NULL);
+        else
+                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC,
+                                                   txn_id, NULL);
+
+        glusterd_op_sm ();
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
new file mode 100644
index 000000000..d5200a4ae
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -0,0 +1,1974 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "rpc-clnt.h"
+#include "glusterd1-xdr.h"
+#include "cli1-xdr.h"
+
+#include "xdr-generic.h"
+
+#include "compat-errno.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-sm.h"
+#include "glusterd.h"
+#include "protocol-common.h"
+#include "glusterd-utils.h"
+#include "common-utils.h"
+#include <sys/uio.h>
+
+
+#define SERVER_PATH_MAX  (16 * 1024)
+
+
+extern glusterd_op_info_t opinfo;
+extern uuid_t global_txn_id;
+
+int32_t
+glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
+                               int32_t op_errno, rpcsvc_request_t *req,
+                               void *op_ctx, char *op_errstr)
+{
+        int32_t         ret = -1;
+        void            *cli_rsp = NULL;
+        dict_t          *ctx = NULL;
+        char            *free_ptr = NULL;
+        glusterd_conf_t *conf = NULL;
+        xdrproc_t       xdrproc = NULL;
+        char            *errstr = NULL;
+        int32_t         status = 0;
+        int32_t         count = 0;
+        gf_cli_rsp      rsp = {0,};
+        xlator_t        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        ctx = op_ctx;
+
+        switch (op) {
+        case GD_OP_REMOVE_BRICK:
+        {
+                if (ctx)
+                        ret = dict_get_str (ctx, "errstr", &errstr);
+                break;
+        }
+        case GD_OP_RESET_VOLUME:
+        {
+                if (op_ret && !op_errstr)
+                        errstr = "Error while resetting options";
+                break;
+        }
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+        {
+                if (ctx) {
+                        ret = dict_get_int32 (ctx, "status", &status);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "failed to get status");
+                        }
+                }
+                break;
+        }
+        case GD_OP_GSYNC_CREATE:
+        case GD_OP_GSYNC_SET:
+        {
+               if (ctx) {
+                        ret = dict_get_str (ctx, "errstr", &errstr);
+                        ret = dict_set_str (ctx, "glusterd_workdir", conf->workdir);
+                        /* swallow error here, that will be re-triggered in cli */
+
+               }
+               break;
+
+        }
+        case GD_OP_QUOTA:
+        {
+                if (ctx && !op_errstr) {
+                        ret = dict_get_str (ctx, "errstr", &errstr);
+                }
+                break;
+        }
+        case GD_OP_PROFILE_VOLUME:
+        {
+                if (ctx && dict_get_int32 (ctx, "count", &count)) {
+                        ret = dict_set_int32 (ctx, "count", 0);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to set count in dictionary");
+                        }
+                }
+                break;
+        }
+        case GD_OP_START_BRICK:
+        case GD_OP_STOP_BRICK:
+        {
+                gf_log (this->name, GF_LOG_DEBUG, "op '%s' not supported",
+                        gd_op_list[op]);
+                break;
+        }
+        case GD_OP_NONE:
+        case GD_OP_MAX:
+        {
+                gf_log (this->name, GF_LOG_ERROR, "invalid operation");
+                break;
+        }
+        case GD_OP_CREATE_VOLUME:
+        case GD_OP_START_VOLUME:
+        case GD_OP_STOP_VOLUME:
+        case GD_OP_DELETE_VOLUME:
+        case GD_OP_DEFRAG_VOLUME:
+        case GD_OP_ADD_BRICK:
+        case GD_OP_LOG_ROTATE:
+        case GD_OP_SYNC_VOLUME:
+        case GD_OP_STATEDUMP_VOLUME:
+        case GD_OP_REPLACE_BRICK:
+        case GD_OP_STATUS_VOLUME:
+        case GD_OP_SET_VOLUME:
+        case GD_OP_LIST_VOLUME:
+        case GD_OP_CLEARLOCKS_VOLUME:
+        case GD_OP_HEAL_VOLUME:
+        case GD_OP_SNAP:
+        {
+                /*nothing specific to be done*/
+                break;
+        }
+        case GD_OP_COPY_FILE:
+        {
+               if (ctx)
+                        ret = dict_get_str (ctx, "errstr", &errstr);
+               break;
+        }
+        case GD_OP_SYS_EXEC:
+        {
+               if (ctx) {
+                        ret = dict_get_str (ctx, "errstr", &errstr);
+                        ret = dict_set_str (ctx, "glusterd_workdir",
+                                            conf->workdir);
+               }
+               break;
+        }
+        }
+
+        rsp.op_ret = op_ret;
+        rsp.op_errno = errno;
+        if (errstr)
+                rsp.op_errstr = errstr;
+        else if (op_errstr)
+                rsp.op_errstr = op_errstr;
+
+        if (!rsp.op_errstr)
+                rsp.op_errstr = "";
+
+        if (ctx) {
+                ret = dict_allocate_and_serialize (ctx, &rsp.dict.dict_val,
+                                                   &rsp.dict.dict_len);
+                if (ret < 0 )
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "serialize buffer");
+                else
+                        free_ptr = rsp.dict.dict_val;
+        }
+
+        /* needed by 'rebalance status' */
+        if (status)
+                rsp.op_errno = status;
+
+        cli_rsp = &rsp;
+        xdrproc = (xdrproc_t) xdr_gf_cli_rsp;
+
+        glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                         xdrproc, ctx);
+        ret = 0;
+
+        GF_FREE (free_ptr);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe, fop_cbk_fn_t fn)
+{
+        glusterd_conf_t *priv = THIS->private;
+        int             ret   = -1;
+
+        synclock_lock (&priv->big_lock);
+        ret = fn (req, iov, count, myframe);
+        synclock_unlock (&priv->big_lock);
+
+        return ret;
+}
+
+int
+__glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        gd1_mgmt_probe_rsp    rsp   = {{0},};
+        int                   ret   = 0;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        glusterd_friend_sm_event_t    *event = NULL;
+        glusterd_probe_ctx_t          *ctx = NULL;
+
+        if (-1 == req->rpc_status) {
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "error");
+                //rsp.op_ret   = -1;
+                //rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO,
+                "Received probe resp from uuid: %s, host: %s",
+                uuid_utoa (rsp.uuid), rsp.hostname);
+        if (rsp.op_ret != 0) {
+                ctx = ((call_frame_t *)myframe)->local;
+                ((call_frame_t *)myframe)->local = NULL;
+
+                GF_ASSERT (ctx);
+
+                if (ctx->req) {
+                        glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
+                                                      rsp.op_errno,
+                                                      rsp.op_errstr,
+                                                      ctx->hostname, ctx->port,
+                                                      ctx->dict);
+                }
+
+                glusterd_destroy_probe_ctx (ctx);
+                (void) glusterd_friend_remove (rsp.uuid, rsp.hostname);
+                ret = rsp.op_ret;
+                goto out;
+        }
+        ret = glusterd_friend_find (rsp.uuid, rsp.hostname, &peerinfo);
+        if (ret) {
+                GF_ASSERT (0);
+        }
+
+        if (strncasecmp (rsp.hostname, peerinfo->hostname, 1024)) {
+                gf_log (THIS->name, GF_LOG_INFO, "Host: %s  with uuid: %s "
+                        "already present in cluster with alias hostname: %s",
+                        rsp.hostname, uuid_utoa (rsp.uuid), peerinfo->hostname);
+
+                ctx = ((call_frame_t *)myframe)->local;
+                ((call_frame_t *)myframe)->local = NULL;
+
+                GF_ASSERT (ctx);
+
+                rsp.op_errno = GF_PROBE_FRIEND;
+                if (ctx->req) {
+                        glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
+                                                      rsp.op_errno,
+                                                      rsp.op_errstr,
+                                                      ctx->hostname, ctx->port,
+                                                      ctx->dict);
+                }
+
+                glusterd_destroy_probe_ctx (ctx);
+                (void) glusterd_friend_remove (NULL, rsp.hostname);
+                ret = rsp.op_ret;
+                goto out;
+        }
+
+        uuid_copy (peerinfo->uuid, rsp.uuid);
+
+        ret = glusterd_friend_sm_new_event
+                        (GD_FRIEND_EVENT_INIT_FRIEND_REQ, &event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                         "Unable to get event");
+                goto out;
+        }
+
+        event->peerinfo = peerinfo;
+        event->ctx      = ((call_frame_t *)myframe)->local;
+        ((call_frame_t *)myframe)->local = NULL;
+        ret = glusterd_friend_sm_inject_event (event);
+
+
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        gf_log ("glusterd", GF_LOG_INFO, "Received resp to probe req");
+
+out:
+        free (rsp.hostname);//malloced by xdr
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int
+glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_probe_cbk);
+}
+
+
+int
+__glusterd_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        gd1_mgmt_friend_rsp           rsp   = {{0},};
+        int                           ret   = -1;
+        glusterd_friend_sm_event_t        *event = NULL;
+        glusterd_friend_sm_event_type_t    event_type = GD_FRIEND_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        int32_t                       op_ret = -1;
+        int32_t                       op_errno = -1;
+        glusterd_probe_ctx_t          *ctx = NULL;
+        glusterd_friend_update_ctx_t  *ev_ctx = NULL;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "error");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+        gf_log ("glusterd", GF_LOG_INFO,
+                "Received %s from uuid: %s, host: %s, port: %d",
+                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid), rsp.hostname, rsp.port);
+
+        ret = glusterd_friend_find (rsp.uuid, rsp.hostname, &peerinfo);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "received friend add response from"
+                        " unknown peer uuid: %s", uuid_utoa (rsp.uuid));
+                goto out;
+        }
+
+        if (op_ret)
+                event_type = GD_FRIEND_EVENT_RCVD_RJT;
+        else
+                event_type = GD_FRIEND_EVENT_RCVD_ACC;
+
+        ret = glusterd_friend_sm_new_event (event_type, &event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                         "Unable to get event");
+                goto out;
+        }
+        event->peerinfo = peerinfo;
+        ev_ctx = GF_CALLOC (1, sizeof (*ev_ctx),
+                                gf_gld_mt_friend_update_ctx_t);
+        if (!ev_ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (ev_ctx->uuid, rsp.uuid);
+        ev_ctx->hostname = gf_strdup (rsp.hostname);
+
+        event->ctx = ev_ctx;
+        ret = glusterd_friend_sm_inject_event (event);
+
+        if (ret)
+                goto out;
+
+out:
+        ctx = ((call_frame_t *)myframe)->local;
+        ((call_frame_t *)myframe)->local = NULL;
+
+        GF_ASSERT (ctx);
+
+        if (ctx->req)//reverse probe doesn't have req
+                ret = glusterd_xfer_cli_probe_resp (ctx->req, op_ret, op_errno,
+                                                    NULL, ctx->hostname,
+                                                    ctx->port, ctx->dict);
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+        if (ctx)
+                glusterd_destroy_probe_ctx (ctx);
+        free (rsp.hostname);//malloced by xdr
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int
+glusterd_friend_add_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_friend_add_cbk);
+}
+
+int
+__glusterd_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
+                               int count, void *myframe)
+{
+        gd1_mgmt_friend_rsp             rsp   = {{0},};
+        glusterd_conf_t                 *conf = NULL;
+        int                             ret   = -1;
+        glusterd_friend_sm_event_t      *event = NULL;
+        glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        int32_t                         op_ret = -1;
+        int32_t                         op_errno = -1;
+        glusterd_probe_ctx_t            *ctx = NULL;
+        gf_boolean_t                    move_sm_now = _gf_true;
+
+        conf  = THIS->private;
+        GF_ASSERT (conf);
+
+        ctx = ((call_frame_t *)myframe)->local;
+        ((call_frame_t *)myframe)->local = NULL;
+        GF_ASSERT (ctx);
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                move_sm_now = _gf_false;
+                goto inject;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "error");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto respond;
+        }
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+        gf_log ("glusterd", GF_LOG_INFO,
+                "Received %s from uuid: %s, host: %s, port: %d",
+                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid), rsp.hostname, rsp.port);
+
+inject:
+        ret = glusterd_friend_find (rsp.uuid, ctx->hostname, &peerinfo);
+
+        if (ret) {
+                //can happen as part of rpc clnt connection cleanup
+                //when the frame timeout happens after 30 minutes
+                goto respond;
+        }
+
+        event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
+
+        ret = glusterd_friend_sm_new_event (event_type, &event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                         "Unable to get event");
+                goto respond;
+        }
+        event->peerinfo = peerinfo;
+
+        ret = glusterd_friend_sm_inject_event (event);
+
+        if (ret)
+                goto respond;
+
+        /*friend_sm would be moved on CLNT_DISCONNECT, consequently
+          cleaning up peerinfo. Else, we run the risk of triggering
+          a clnt_destroy within saved_frames_unwind.
+        */
+        op_ret = 0;
+
+
+respond:
+        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, op_ret, op_errno, NULL,
+                                              ctx->hostname, ctx->dict);
+        if (!ret && move_sm_now) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        if (ctx) {
+                glusterd_broadcast_friend_delete (ctx->hostname, NULL);
+                glusterd_destroy_probe_ctx (ctx);
+        }
+
+        free (rsp.hostname);//malloced by xdr
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int
+glusterd_friend_remove_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_friend_remove_cbk);
+}
+
+int32_t
+__glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
+{
+        int                           ret    = -1;
+        gd1_mgmt_friend_update_rsp    rsp    = {{0}, };
+        xlator_t                      *this  = NULL;
+
+        GF_ASSERT (req);
+        this = THIS;
+
+        if (-1 == req->rpc_status) {
+                gf_log (this->name, GF_LOG_ERROR, "RPC Error");
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize friend"
+                        " update repsonse");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_INFO, "Received %s from uuid: %s",
+                (ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int
+glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
+                    int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_friend_update_cbk);
+}
+
+int32_t
+__glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
+{
+        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = &global_txn_id;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+out:
+        op_ret = rsp.op_ret;
+
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received lock %s from uuid: %s", (op_ret) ? "RJT" : "ACC",
+                uuid_utoa (rsp.uuid));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Lock response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
+        }
+
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
+
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int32_t
+glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_cluster_lock_cbk);
+}
+
+static int32_t
+glusterd_mgmt_v3_lock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                    int count, void *myframe)
+{
+        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode mgmt_v3 lock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+out:
+        op_ret = rsp.op_ret;
+
+        txn_id = &rsp.txn_id;
+
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received mgmt_v3 lock %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "mgmt_v3 lock response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
+        }
+
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
+
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_lock_peers_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        glusterd_mgmt_v3_lock_peers_cbk_fn);
+}
+
+static int32_t
+glusterd_mgmt_v3_unlock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                      int count, void *myframe)
+{
+        gd1_mgmt_v3_unlock_rsp        rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to decode mgmt_v3 unlock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+out:
+        op_ret = rsp.op_ret;
+
+        txn_id = &rsp.txn_id;
+
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received mgmt_v3 unlock %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC",
+                uuid_utoa (rsp.uuid));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "mgmt_v3 unlock response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
+        }
+
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
+                                              "progress. Please try again after"
+                                              " sometime.");
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
+
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_unlock_peers_cbk (struct rpc_req *req, struct iovec *iov,
+                                   int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        glusterd_mgmt_v3_unlock_peers_cbk_fn);
+}
+
+int32_t
+__glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = &global_txn_id;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                goto out;
+        }
+
+out:
+        op_ret = rsp.op_ret;
+
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received unlock %s from uuid: %s",
+                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Unlock response received "
+                        "from unknown peer %s", uuid_utoa (rsp.uuid));
+        }
+
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
+
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int32_t
+glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_cluster_unlock_cbk);
+}
+
+int32_t
+__glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        gd1_mgmt_stage_op_rsp         rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        dict_t                        *dict   = NULL;
+        char                          err_str[2048] = {0};
+        char                          *peer_str = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("error");
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode stage "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("Failed to decode stage response "
+                                        "received from peer.");
+                goto out;
+        }
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize rsp-buffer to dictionary");
+			event_type = GD_OP_EVENT_RCVD_RJT;
+                        goto out;
+                } else {
+                        dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+out:
+        op_ret = rsp.op_ret;
+
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received stage %s from uuid: %s",
+                (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
+
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Stage response received "
+                        "from unknown peer: %s", uuid_utoa (rsp.uuid));
+        }
+
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+                if (strcmp ("", rsp.op_errstr)) {
+                        opinfo.op_errstr = gf_strdup (rsp.op_errstr);
+                } else {
+                        if (peerinfo)
+                                peer_str = peerinfo->hostname;
+                        else
+                                peer_str = uuid_utoa (rsp.uuid);
+                        snprintf (err_str, sizeof (err_str),
+                                  OPERRSTR_STAGE_FAIL, peer_str);
+                        opinfo.op_errstr = gf_strdup (err_str);
+                }
+                if (!opinfo.op_errstr) {
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
+
+        switch (rsp.op) {
+        case GD_OP_REPLACE_BRICK:
+                glusterd_rb_use_rsp_dict (NULL, dict);
+                break;
+        }
+
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        free (rsp.op_errstr); //malloced by xdr
+        if (dict) {
+                if (!dict->extra_stdfree && rsp.dict.dict_val)
+                        free (rsp.dict.dict_val); //malloced by xdr
+                dict_unref (dict);
+        } else {
+                free (rsp.dict.dict_val); //malloced by xdr
+        }
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int32_t
+glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_stage_op_cbk);
+}
+
+int32_t
+__glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        gd1_mgmt_commit_op_rsp         rsp   = {{0},};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        glusterd_peerinfo_t           *peerinfo = NULL;
+        dict_t                        *dict = NULL;
+        char                          err_str[2048] = {0};
+        char                          *peer_str = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = NULL;
+
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("error");
+		event_type = GD_OP_EVENT_RCVD_RJT;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
+                        "response received from peer");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("Failed to decode commit response "
+                                        "received from peer.");
+		event_type = GD_OP_EVENT_RCVD_RJT;
+                goto out;
+        }
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize rsp-buffer to dictionary");
+			event_type = GD_OP_EVENT_RCVD_RJT;
+                        goto out;
+                } else {
+                        dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        op_ret = rsp.op_ret;
+
+        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
+                "Received commit %s from uuid: %s",
+                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+
+        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Commit response for "
+                        "'Volume %s' received from unknown peer: %s",
+                        gd_op_list[opinfo.op], uuid_utoa (rsp.uuid));
+        }
+
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                opinfo.op_ret = op_ret;
+                if (strcmp ("", rsp.op_errstr)) {
+                        opinfo.op_errstr = gf_strdup(rsp.op_errstr);
+                } else {
+                        if (peerinfo)
+                                peer_str = peerinfo->hostname;
+                        else
+                                peer_str = uuid_utoa (rsp.uuid);
+                        snprintf (err_str, sizeof (err_str),
+                                  OPERRSTR_COMMIT_FAIL, peer_str);
+                        opinfo.op_errstr = gf_strdup (err_str);
+                }
+                if (!opinfo.op_errstr) {
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+                switch (rsp.op) {
+                case GD_OP_REPLACE_BRICK:
+                        ret = glusterd_rb_use_rsp_dict (NULL, dict);
+                        if (ret)
+                                goto out;
+                break;
+
+                case GD_OP_SYNC_VOLUME:
+                        ret = glusterd_sync_use_rsp_dict (NULL, dict);
+                        if (ret)
+                                goto out;
+                break;
+
+                case GD_OP_PROFILE_VOLUME:
+                        ret = glusterd_profile_volume_use_rsp_dict (NULL, dict);
+                        if (ret)
+                                goto out;
+                break;
+
+                case GD_OP_GSYNC_SET:
+                        ret = glusterd_gsync_use_rsp_dict (NULL, dict, rsp.op_errstr);
+                        if (ret)
+                                goto out;
+                break;
+
+                case GD_OP_STATUS_VOLUME:
+                        ret = glusterd_volume_status_copy_to_op_ctx_dict (NULL, dict);
+                        if (ret)
+                                goto out;
+                break;
+
+                case GD_OP_REBALANCE:
+                case GD_OP_DEFRAG_BRICK_VOLUME:
+                        ret = glusterd_volume_rebalance_use_rsp_dict (NULL, dict);
+                        if (ret)
+                                goto out;
+                break;
+
+                case GD_OP_HEAL_VOLUME:
+                        ret = glusterd_volume_heal_use_rsp_dict (NULL, dict);
+                        if (ret)
+                                goto out;
+
+                break;
+
+                default:
+                break;
+                }
+        }
+
+out:
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        if (dict)
+                dict_unref (dict);
+        free (rsp.op_errstr); //malloced by xdr
+        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
+        return ret;
+}
+
+int32_t
+glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_commit_op_cbk);
+}
+
+int32_t
+glusterd_rpc_probe (call_frame_t *frame, xlator_t *this,
+                    void *data)
+{
+        gd1_mgmt_probe_req      req = {{0},};
+        int                     ret = 0;
+        int                     port = 0;
+        char                    *hostname = NULL;
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        glusterd_conf_t         *priv = NULL;
+        dict_t                  *dict = NULL;
+
+        if (!frame || !this ||  !data) {
+                ret = -1;
+                goto out;
+        }
+
+        dict = data;
+        priv = this->private;
+
+        GF_ASSERT (priv);
+        ret = dict_get_str (dict, "hostname", &hostname);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (dict, "port", &port);
+        if (ret)
+                port = GF_DEFAULT_BASE_PORT;
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, MY_UUID);
+        req.hostname = gf_strdup (hostname);
+        req.port = port;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
+                                       GLUSTERD_PROBE_QUERY,
+                                       NULL, this, glusterd_probe_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_probe_req);
+
+out:
+        GF_FREE (req.hostname);
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+int32_t
+glusterd_rpc_friend_add (call_frame_t *frame, xlator_t *this,
+                         void *data)
+{
+        gd1_mgmt_friend_req         req      = {{0},};
+        int                         ret      = 0;
+        glusterd_peerinfo_t        *peerinfo = NULL;
+        glusterd_conf_t            *priv     = NULL;
+        glusterd_friend_sm_event_t *event    = NULL;
+        dict_t                     *vols     = NULL;
+
+
+        if (!frame || !this || !data) {
+                ret = -1;
+                goto out;
+        }
+
+        event = data;
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        peerinfo = event->peerinfo;
+
+        ret = glusterd_build_volume_dict (&vols);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, MY_UUID);
+        req.hostname = peerinfo->hostname;
+        req.port = peerinfo->port;
+
+        ret = dict_allocate_and_serialize (vols, &req.vols.vols_val,
+                                           &req.vols.vols_len);
+        if (ret)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
+                                       GLUSTERD_FRIEND_ADD,
+                                       NULL, this, glusterd_friend_add_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_friend_req);
+
+
+out:
+        GF_FREE (req.vols.vols_val);
+
+        if (vols)
+                dict_unref (vols);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_rpc_friend_remove (call_frame_t *frame, xlator_t *this,
+                            void *data)
+{
+        gd1_mgmt_friend_req             req = {{0},};
+        int                             ret = 0;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_conf_t                 *priv = NULL;
+        glusterd_friend_sm_event_t      *event = NULL;
+
+        if (!frame || !this || !data) {
+                ret = -1;
+                goto out;
+        }
+
+        event = data;
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        peerinfo = event->peerinfo;
+
+        uuid_copy (req.uuid, MY_UUID);
+        req.hostname = peerinfo->hostname;
+        req.port = peerinfo->port;
+        ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
+                                       GLUSTERD_FRIEND_REMOVE, NULL,
+                                       this, glusterd_friend_remove_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_friend_req);
+
+out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+int32_t
+glusterd_rpc_friend_update (call_frame_t *frame, xlator_t *this,
+                            void *data)
+{
+        gd1_mgmt_friend_update  req         = {{0},};
+        int                     ret         = 0;
+        glusterd_conf_t        *priv        = NULL;
+        dict_t                 *friends     = NULL;
+        call_frame_t           *dummy_frame = NULL;
+        glusterd_peerinfo_t    *peerinfo    = NULL;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        friends = data;
+        if (!friends)
+                goto out;
+
+        ret = dict_get_ptr (friends, "peerinfo", VOID(&peerinfo));
+        if (ret)
+                goto out;
+
+        ret = dict_allocate_and_serialize (friends, &req.friends.friends_val,
+                                           &req.friends.friends_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, MY_UUID);
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->peer,
+                                       GLUSTERD_FRIEND_UPDATE, NULL,
+                                       this, glusterd_friend_update_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_friend_update);
+
+out:
+        GF_FREE (req.friends.friends_val);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_cluster_lock (call_frame_t *frame, xlator_t *this,
+                          void *data)
+{
+        gd1_mgmt_cluster_lock_req       req = {{0},};
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_conf_t                 *priv = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+
+        if (!this)
+                goto out;
+
+        peerinfo = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        glusterd_get_uuid (&req.uuid);
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_LOCK,
+                                       NULL,
+                                       this, glusterd_cluster_lock_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_lock_peers (call_frame_t *frame, xlator_t *this,
+                             void *data)
+{
+        gd1_mgmt_v3_lock_req             req         = {{0},};
+        int                              ret         = -1;
+        glusterd_peerinfo_t             *peerinfo    = NULL;
+        glusterd_conf_t                 *priv        = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        dict_t                          *dict        = NULL;
+        uuid_t                          *txn_id      = NULL;
+
+        if (!this)
+                goto out;
+
+        dict = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
+                goto out;
+        }
+
+        /* Sending valid transaction ID to peers */
+        ret = dict_get_bin (dict, "transaction_id",
+                            (void **)&txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to get transaction id.");
+                goto out;
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+                uuid_copy (req.txn_id, *txn_id);
+        }
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt_v3,
+                                       GLUSTERD_MGMT_V3_LOCK, NULL,
+                                       this, glusterd_mgmt_v3_lock_peers_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_mgmt_v3_unlock_peers (call_frame_t *frame, xlator_t *this,
+                               void *data)
+{
+        gd1_mgmt_v3_unlock_req           req         = {{0},};
+        int                              ret         = -1;
+        glusterd_peerinfo_t             *peerinfo    = NULL;
+        glusterd_conf_t                 *priv        = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        dict_t                          *dict        = NULL;
+        uuid_t                          *txn_id      = NULL;
+
+        if (!this)
+                goto out;
+
+        dict = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+
+        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
+                goto out;
+        }
+
+        /* Sending valid transaction ID to peers */
+        ret = dict_get_bin (dict, "transaction_id",
+                            (void **)&txn_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                       "Failed to get transaction id.");
+                goto out;
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+                uuid_copy (req.txn_id, *txn_id);
+        }
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt_v3,
+                                       GLUSTERD_MGMT_V3_UNLOCK, NULL,
+                                       this, glusterd_mgmt_v3_unlock_peers_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_cluster_unlock (call_frame_t *frame, xlator_t *this,
+                            void *data)
+{
+        gd1_mgmt_cluster_lock_req       req = {{0},};
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_conf_t                 *priv = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+
+        if (!this ) {
+                ret = -1;
+                goto out;
+        }
+        peerinfo = data;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        glusterd_get_uuid (&req.uuid);
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_UNLOCK,
+                                       NULL,
+                                       this, glusterd_cluster_unlock_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_stage_op (call_frame_t *frame, xlator_t *this,
+                      void *data)
+{
+        gd1_mgmt_stage_op_req           req = {{0,},};
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_conf_t                 *priv = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        dict_t                          *dict = NULL;
+        gf_boolean_t                    is_alloc = _gf_true;
+
+        if (!this) {
+                goto out;
+        }
+
+        dict = data;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+        req.op = glusterd_op_get_op ();
+
+        ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
+                                           &req.buf.buf_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
+                        "to request buffer");
+                goto out;
+        }
+
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt, GLUSTERD_MGMT_STAGE_OP,
+                                       NULL,
+                                       this, glusterd_stage_op_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
+
+out:
+        if ((_gf_true == is_alloc) && req.buf.buf_val)
+                GF_FREE (req.buf.buf_val);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_commit_op (call_frame_t *frame, xlator_t *this,
+                      void *data)
+{
+        gd1_mgmt_commit_op_req  req         = {{0,},};
+        int                     ret         = -1;
+        glusterd_peerinfo_t    *peerinfo    = NULL;
+        glusterd_conf_t        *priv        = NULL;
+        call_frame_t           *dummy_frame = NULL;
+        dict_t                 *dict        = NULL;
+        gf_boolean_t            is_alloc    = _gf_true;
+
+        if (!this) {
+                goto out;
+        }
+
+        dict = data;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
+        if (ret)
+                goto out;
+
+        //peerinfo should not be in payload
+        dict_del (dict, "peerinfo");
+
+        glusterd_get_uuid (&req.uuid);
+        req.op = glusterd_op_get_op ();
+
+        ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
+                                           &req.buf.buf_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict to "
+                        "request buffer");
+                goto out;
+        }
+
+        dummy_frame = create_frame (this, this->ctx->pool);
+        if (!dummy_frame)
+                goto out;
+
+        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
+                                       peerinfo->mgmt, GLUSTERD_MGMT_COMMIT_OP,
+                                       NULL,
+                                       this, glusterd_commit_op_cbk,
+                                       (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
+
+out:
+        if ((_gf_true == is_alloc) && req.buf.buf_val)
+                GF_FREE (req.buf.buf_val);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+__glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        gd1_mgmt_brick_op_rsp         rsp   = {0};
+        int                           ret   = -1;
+        int32_t                       op_ret = -1;
+        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
+        call_frame_t                  *frame = NULL;
+        glusterd_op_brick_rsp_ctx_t   *ev_ctx = NULL;
+        dict_t                        *dict = NULL;
+        int                           index = 0;
+        glusterd_req_ctx_t            *req_ctx = NULL;
+        glusterd_pending_node_t       *node = NULL;
+        xlator_t                      *this = NULL;
+        uuid_t                        *txn_id = &global_txn_id;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (req);
+        frame = myframe;
+        req_ctx = frame->local;
+
+        if (-1 == req->rpc_status) {
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                /* use standard allocation because to keep uniformity
+                   in freeing it */
+                rsp.op_errstr = strdup ("error");
+		event_type = GD_OP_EVENT_RCVD_RJT;
+                goto out;
+        }
+
+        ret =  xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to decode brick op "
+                        "response received");
+                rsp.op_ret   = -1;
+                rsp.op_errno = EINVAL;
+                rsp.op_errstr = strdup ("Unable to decode brick op response");
+		event_type = GD_OP_EVENT_RCVD_RJT;
+                goto out;
+        }
+
+        if (rsp.output.output_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.output.output_val,
+                                        rsp.output.output_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "unserialize rsp-buffer to dictionary");
+			event_type = GD_OP_EVENT_RCVD_RJT;
+                        goto out;
+                } else {
+                        dict->extra_stdfree = rsp.output.output_val;
+                }
+        }
+
+        op_ret = rsp.op_ret;
+
+        /* Add index to rsp_dict for GD_OP_STATUS_VOLUME */
+        if (GD_OP_STATUS_VOLUME == req_ctx->op) {
+                node = frame->cookie;
+                index = node->index;
+                ret = dict_set_int32 (dict, "index", index);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting index on brick status rsp dict");
+                        rsp.op_ret = -1;
+                        event_type = GD_OP_EVENT_RCVD_RJT;
+                        goto out;
+                }
+        }
+out:
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        ev_ctx = GF_CALLOC (1, sizeof (*ev_ctx), gf_gld_mt_brick_rsp_ctx_t);
+        GF_ASSERT (ev_ctx);
+        if (op_ret) {
+                event_type = GD_OP_EVENT_RCVD_RJT;
+                ev_ctx->op_ret = op_ret;
+                ev_ctx->op_errstr = gf_strdup(rsp.op_errstr);
+        } else {
+                event_type = GD_OP_EVENT_RCVD_ACC;
+        }
+        ev_ctx->pending_node = frame->cookie;
+        ev_ctx->rsp_dict  = dict;
+        ev_ctx->commit_ctx = frame->local;
+        ret = glusterd_op_sm_inject_event (event_type, txn_id, ev_ctx);
+        if (!ret) {
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+        }
+
+        if (ret && dict)
+                dict_unref (dict);
+        free (rsp.op_errstr); //malloced by xdr
+        GLUSTERD_STACK_DESTROY (frame);
+        return ret;
+}
+
+int32_t
+glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        __glusterd_brick_op_cbk);
+}
+
+int32_t
+glusterd_brick_op (call_frame_t *frame, xlator_t *this,
+                      void *data)
+{
+
+        gd1_mgmt_brick_op_req           *req = NULL;
+        int                             ret = 0;
+        glusterd_conf_t                 *priv = NULL;
+        call_frame_t                    *dummy_frame = NULL;
+        char                            *op_errstr = NULL;
+        int                             pending_bricks = 0;
+        glusterd_pending_node_t         *pending_node;
+        glusterd_req_ctx_t              *req_ctx = NULL;
+        struct rpc_clnt                 *rpc = NULL;
+        dict_t                          *op_ctx = NULL;
+        uuid_t                          *txn_id = &global_txn_id;
+
+        if (!this) {
+                ret = -1;
+                goto out;
+        }
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        req_ctx = data;
+        GF_ASSERT (req_ctx);
+        INIT_LIST_HEAD (&opinfo.pending_bricks);
+        ret = glusterd_op_bricks_select (req_ctx->op, req_ctx->dict, &op_errstr,
+                                         &opinfo.pending_bricks, NULL);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to select bricks "
+                        "while performing brick op during 'Volume %s'",
+                        gd_op_list[opinfo.op]);
+                opinfo.op_errstr = op_errstr;
+                goto out;
+        }
+
+
+        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
+
+        gf_log ("", GF_LOG_DEBUG, "transaction ID = %s", uuid_utoa (*txn_id));
+
+        list_for_each_entry (pending_node, &opinfo.pending_bricks, list) {
+                dummy_frame = create_frame (this, this->ctx->pool);
+                if (!dummy_frame)
+                        continue;
+
+                if ((pending_node->type == GD_NODE_NFS) ||
+                    ((pending_node->type == GD_NODE_SHD) &&
+                    (req_ctx->op == GD_OP_STATUS_VOLUME)))
+                        ret = glusterd_node_op_build_payload
+                                (req_ctx->op,
+                                 (gd1_mgmt_brick_op_req **)&req,
+                                 req_ctx->dict);
+                else {
+                        ret = glusterd_brick_op_build_payload
+                                (req_ctx->op, pending_node->node,
+                                 (gd1_mgmt_brick_op_req **)&req,
+                                 req_ctx->dict);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "build brick op payload during "
+                                        "'Volume %s'", gd_op_list[req_ctx->op]);
+                                goto out;
+                        }
+                }
+
+                dummy_frame->local = data;
+                dummy_frame->cookie = pending_node;
+
+                rpc = glusterd_pending_node_get_rpc (pending_node);
+                if (!rpc) {
+                        if (pending_node->type == GD_NODE_REBALANCE) {
+                                opinfo.brick_pending_count = 0;
+                                ret = 0;
+                                if (req) {
+                                        GF_FREE (req->input.input_val);
+                                        GF_FREE (req);
+                                        req = NULL;
+                                }
+                                GLUSTERD_STACK_DESTROY (dummy_frame);
+
+                                op_ctx = glusterd_op_get_ctx ();
+                                if (!op_ctx)
+                                        goto out;
+                                glusterd_defrag_volume_node_rsp (req_ctx->dict,
+                                                                 NULL, op_ctx);
+
+                                goto out;
+                        }
+
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR, "Brick Op failed "
+                                "due to rpc failure.");
+                        goto out;
+                }
+
+                ret = glusterd_submit_request (rpc, req, dummy_frame,
+                                               priv->gfs_mgmt,
+                                               req->op, NULL,
+                                               this, glusterd_brick_op_cbk,
+                                               (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+                if (req) {
+                        GF_FREE (req->input.input_val);
+                        GF_FREE (req);
+                        req = NULL;
+                }
+                if (!ret)
+                        pending_bricks++;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for operation "
+                "'Volume %s' to %d bricks", gd_op_list[req_ctx->op],
+                pending_bricks);
+        opinfo.brick_pending_count = pending_bricks;
+
+out:
+        if (ret) {
+                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
+                                             txn_id, data);
+                opinfo.op_ret = ret;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+struct rpc_clnt_procedure gd_brick_actors[GLUSTERD_BRICK_MAXVALUE] = {
+        [GLUSTERD_BRICK_NULL] = {"NULL", NULL },
+        [GLUSTERD_BRICK_OP]   = {"BRICK_OP", glusterd_brick_op},
+};
+
+struct rpc_clnt_procedure gd_peer_actors[GLUSTERD_FRIEND_MAXVALUE] = {
+        [GLUSTERD_FRIEND_NULL]   = {"NULL", NULL },
+        [GLUSTERD_PROBE_QUERY]   = {"PROBE_QUERY", glusterd_rpc_probe},
+        [GLUSTERD_FRIEND_ADD]    = {"FRIEND_ADD", glusterd_rpc_friend_add},
+        [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE", glusterd_rpc_friend_remove},
+        [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd_rpc_friend_update},
+};
+
+struct rpc_clnt_procedure gd_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
+        [GLUSTERD_MGMT_NULL]           = {"NULL", NULL },
+        [GLUSTERD_MGMT_CLUSTER_LOCK]   = {"CLUSTER_LOCK", glusterd_cluster_lock},
+        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK", glusterd_cluster_unlock},
+        [GLUSTERD_MGMT_STAGE_OP]       = {"STAGE_OP", glusterd_stage_op},
+        [GLUSTERD_MGMT_COMMIT_OP]      = {"COMMIT_OP", glusterd_commit_op},
+};
+
+struct rpc_clnt_procedure gd_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
+        [GLUSTERD_MGMT_V3_NULL]           = {"NULL", NULL },
+        [GLUSTERD_MGMT_V3_LOCK]           = {"MGMT_V3_LOCK", glusterd_mgmt_v3_lock_peers},
+        [GLUSTERD_MGMT_V3_UNLOCK]         = {"MGMT_V3_UNLOCK", glusterd_mgmt_v3_unlock_peers},
+};
+
+struct rpc_clnt_program gd_mgmt_prog = {
+        .progname  = "glusterd mgmt",
+        .prognum   = GD_MGMT_PROGRAM,
+        .progver   = GD_MGMT_VERSION,
+        .proctable = gd_mgmt_actors,
+        .numproc   = GLUSTERD_MGMT_MAXVALUE,
+};
+
+struct rpc_clnt_program gd_brick_prog = {
+        .progname  = "brick operations",
+        .prognum   = GD_BRICK_PROGRAM,
+        .progver   = GD_BRICK_VERSION,
+        .proctable = gd_brick_actors,
+        .numproc   = GLUSTERD_BRICK_MAXVALUE,
+};
+
+struct rpc_clnt_program gd_peer_prog = {
+        .progname  = "Peer mgmt",
+        .prognum   = GD_FRIEND_PROGRAM,
+        .progver   = GD_FRIEND_VERSION,
+        .proctable = gd_peer_actors,
+        .numproc   = GLUSTERD_FRIEND_MAXVALUE,
+};
+
+struct rpc_clnt_program gd_mgmt_v3_prog = {
+        .progname  = "glusterd mgmt v3",
+        .prognum   = GD_MGMT_V3_PROGRAM,
+        .progver   = GD_MGMT_V3_VERSION,
+        .proctable = gd_mgmt_v3_actors,
+        .numproc   = GLUSTERD_MGMT_V3_MAXVALUE,
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
new file mode 100644
index 000000000..c671edf68
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -0,0 +1,1109 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <time.h>
+#include <sys/uio.h>
+#include <sys/resource.h>
+
+#include <libgen.h>
+#include "uuid.h"
+
+#include "fnmatch.h"
+#include "xlator.h"
+#include "protocol-common.h"
+#include "glusterd.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "list.h"
+#include "dict.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "statedump.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+
+static struct list_head gd_friend_sm_queue;
+
+static  char *glusterd_friend_sm_state_names[] = {
+        "Establishing Connection",
+        "Probe Sent to Peer",
+        "Probe Received from Peer",
+        "Peer in Cluster",
+        "Accepted peer request",
+        "Sent and Received peer request",
+        "Peer Rejected",
+        "Peer detach in progress",
+        "Probe Received from peer",
+        "Connected to Peer",
+        "Peer is connected and Accepted",
+        "Invalid State"
+};
+
+static char *glusterd_friend_sm_event_names[] = {
+        "GD_FRIEND_EVENT_NONE",
+        "GD_FRIEND_EVENT_PROBE",
+        "GD_FRIEND_EVENT_INIT_FRIEND_REQ",
+        "GD_FRIEND_EVENT_RCVD_ACC",
+        "GD_FRIEND_EVENT_LOCAL_ACC",
+        "GD_FRIEND_EVENT_RCVD_RJT",
+        "GD_FRIEND_EVENT_LOCAL_RJT",
+        "GD_FRIEND_EVENT_RCVD_FRIEND_REQ",
+        "GD_FRIEND_EVENT_INIT_REMOVE_FRIEND",
+        "GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND",
+        "GD_FRIEND_EVENT_REMOVE_FRIEND",
+        "GD_FRIEND_EVENT_CONNECTED",
+        "GD_FRIEND_EVENT_MAX"
+};
+
+char*
+glusterd_friend_sm_state_name_get (int state)
+{
+        if (state < 0 || state >= GD_FRIEND_STATE_MAX)
+                return glusterd_friend_sm_state_names[GD_FRIEND_STATE_MAX];
+        return glusterd_friend_sm_state_names[state];
+}
+
+char*
+glusterd_friend_sm_event_name_get (int event)
+{
+        if (event < 0 || event >= GD_FRIEND_EVENT_MAX)
+                return glusterd_friend_sm_event_names[GD_FRIEND_EVENT_MAX];
+        return glusterd_friend_sm_event_names[event];
+}
+
+void
+glusterd_destroy_probe_ctx (glusterd_probe_ctx_t *ctx)
+{
+        if (!ctx)
+                return;
+
+        GF_FREE (ctx->hostname);
+        GF_FREE (ctx);
+}
+
+void
+glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx)
+{
+        if (!ctx)
+                return;
+
+        if (ctx->vols)
+                dict_unref (ctx->vols);
+        GF_FREE (ctx->hostname);
+        GF_FREE (ctx);
+}
+
+void
+glusterd_destroy_friend_update_ctx (glusterd_friend_update_ctx_t *ctx)
+{
+        if (!ctx)
+                return;
+        GF_FREE (ctx->hostname);
+        GF_FREE (ctx);
+}
+
+int
+glusterd_broadcast_friend_delete (char *hostname, uuid_t uuid)
+{
+        int                             ret = 0;
+        rpc_clnt_procedure_t            *proc = NULL;
+        xlator_t                        *this = NULL;
+        glusterd_friend_update_ctx_t    ctx = {{0},};
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_conf_t                 *priv = NULL;
+        dict_t                          *friends = NULL;
+        char                            key[100] = {0,};
+        int32_t                         count = 0;
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        ctx.hostname = hostname;
+        ctx.op = GD_FRIEND_UPDATE_DEL;
+
+        friends = dict_new ();
+        if (!friends)
+                goto out;
+
+        snprintf (key, sizeof (key), "op");
+        ret = dict_set_int32 (friends, key, ctx.op);
+        if (ret)
+                goto out;
+
+        snprintf (key, sizeof (key), "hostname");
+        ret = dict_set_str (friends, key, hostname);
+        if (ret)
+                goto out;
+
+        ret = dict_set_int32 (friends, "count", count);
+        if (ret)
+                goto out;
+
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                if (!peerinfo->connected || !peerinfo->peer)
+                        continue;
+
+                ret = dict_set_static_ptr (friends, "peerinfo", peerinfo);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
+                        goto out;
+                }
+
+                proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
+                if (proc->fn) {
+                        ret = proc->fn (NULL, this, friends);
+                }
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+out:
+        if (friends)
+                dict_unref (friends);
+
+        return ret;
+}
+
+
+static int
+glusterd_ac_none (glusterd_friend_sm_event_t *event, void *ctx)
+{
+        int ret = 0;
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_ac_error (glusterd_friend_sm_event_t *event, void *ctx)
+{
+        int ret = 0;
+
+        gf_log ("", GF_LOG_ERROR, "Received event %d ", event->event);
+
+        return ret;
+}
+
+static int
+glusterd_ac_reverse_probe_begin (glusterd_friend_sm_event_t *event, void *ctx)
+{
+        int ret = 0;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_friend_sm_event_t      *new_event = NULL;
+        glusterd_probe_ctx_t            *new_ev_ctx = NULL;
+
+        GF_ASSERT (event);
+        GF_ASSERT (ctx);
+
+        peerinfo = event->peerinfo;
+        ret = glusterd_friend_sm_new_event
+                (GD_FRIEND_EVENT_PROBE, &new_event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get new new_event");
+                ret = -1;
+                goto out;
+        }
+
+        new_ev_ctx = GF_CALLOC (1, sizeof(*new_ev_ctx), gf_gld_mt_probe_ctx_t);
+
+        if (!new_ev_ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        new_ev_ctx->hostname = gf_strdup (peerinfo->hostname);
+        new_ev_ctx->port = peerinfo->port;
+        new_ev_ctx->req = NULL;
+
+        new_event->peerinfo = peerinfo;
+        new_event->ctx = new_ev_ctx;
+
+        ret = glusterd_friend_sm_inject_event (new_event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject new_event %d, "
+                                "ret = %d", new_event->event, ret);
+        }
+
+out:
+        if (ret) {
+                GF_FREE (new_event);
+                GF_FREE (new_ev_ctx->hostname);
+                GF_FREE (new_ev_ctx);
+        }
+        gf_log ("", GF_LOG_DEBUG, "returning with %d", ret);
+        return ret;
+}
+
+static int
+glusterd_ac_friend_add (glusterd_friend_sm_event_t *event, void *ctx)
+{
+        int                     ret = 0;
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        glusterd_conf_t         *conf = NULL;
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (event);
+        peerinfo = event->peerinfo;
+
+        this = THIS;
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        if (!peerinfo->peer)
+                goto out;
+        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_ADD];
+        if (proc->fn) {
+                frame = create_frame (this, this->ctx->pool);
+                if (!frame) {
+                        goto out;
+                }
+                frame->local = ctx;
+                ret = proc->fn (frame, this, event);
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_ac_friend_probe (glusterd_friend_sm_event_t *event, void *ctx)
+{
+        int                     ret = -1;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        glusterd_conf_t         *conf = NULL;
+        xlator_t                *this = NULL;
+        glusterd_probe_ctx_t    *probe_ctx = NULL;
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        dict_t                  *dict = NULL;
+
+        GF_ASSERT (ctx);
+
+        probe_ctx = ctx;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        ret = glusterd_friend_find (NULL, probe_ctx->hostname, &peerinfo);
+        if (ret) {
+                //We should not reach this state ideally
+                GF_ASSERT (0);
+                goto out;
+        }
+
+        if (!peerinfo->peer)
+                goto out;
+        proc = &peerinfo->peer->proctable[GLUSTERD_PROBE_QUERY];
+        if (proc->fn) {
+                frame = create_frame (this, this->ctx->pool);
+                if (!frame) {
+                        goto out;
+                }
+                frame->local = ctx;
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+                ret = dict_set_str (dict, "hostname", probe_ctx->hostname);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_int32 (dict, "port", probe_ctx->port);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
+                        goto out;
+                }
+
+                ret = proc->fn (frame, this, dict);
+                if (ret)
+                        goto out;
+
+        }
+
+
+out:
+        if (dict)
+                dict_unref (dict);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_ac_send_friend_remove_req (glusterd_friend_sm_event_t *event,
+                                    void *data)
+{
+        int                     ret = 0;
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        rpc_clnt_procedure_t    *proc = NULL;
+        call_frame_t            *frame = NULL;
+        glusterd_conf_t         *conf = NULL;
+        xlator_t                *this = NULL;
+        glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+        glusterd_probe_ctx_t            *ctx = NULL;
+        glusterd_friend_sm_event_t      *new_event = NULL;
+
+        GF_ASSERT (event);
+        peerinfo = event->peerinfo;
+
+        this = THIS;
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        ctx = event->ctx;
+
+        if (!peerinfo->connected) {
+                event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
+
+                ret = glusterd_friend_sm_new_event (event_type, &new_event);
+
+                if (!ret) {
+                        new_event->peerinfo = peerinfo;
+                        ret = glusterd_friend_sm_inject_event (new_event);
+                } else {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                 "Unable to get event");
+                }
+
+                if (ctx)
+                        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, ret, 0,
+                                                              NULL,
+                                                              ctx->hostname,
+                                                              ctx->dict);
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
+
+                if (ctx) {
+                        glusterd_broadcast_friend_delete (ctx->hostname, NULL);
+                        glusterd_destroy_probe_ctx (ctx);
+                }
+                goto out;
+        }
+
+        if (!peerinfo->peer)
+                goto out;
+        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_REMOVE];
+        if (proc->fn) {
+                frame = create_frame (this, this->ctx->pool);
+                if (!frame) {
+                        goto out;
+                }
+                frame->local = data;
+                ret = proc->fn (frame, this, event);
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+static gf_boolean_t
+glusterd_should_update_peer (glusterd_peerinfo_t *peerinfo,
+                             glusterd_peerinfo_t *cur_peerinfo)
+{
+        gf_boolean_t is_valid = _gf_false;
+
+        if ((peerinfo == cur_peerinfo) ||
+            (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED))
+                is_valid = _gf_true;
+
+        return is_valid;
+}
+
+static int
+glusterd_ac_send_friend_update (glusterd_friend_sm_event_t *event, void *ctx)
+{
+        int                           ret               = 0;
+        glusterd_peerinfo_t          *cur_peerinfo      = NULL;
+        glusterd_peerinfo_t          *peerinfo          = NULL;
+        rpc_clnt_procedure_t         *proc              = NULL;
+        xlator_t                     *this              = NULL;
+        glusterd_friend_update_ctx_t  ev_ctx            = {{0}};
+        glusterd_conf_t              *priv              = NULL;
+        dict_t                       *friends           = NULL;
+        char                          key[100]          = {0,};
+        char                         *dup_buf           = NULL;
+        int32_t                       count             = 0;
+
+        GF_ASSERT (event);
+        cur_peerinfo = event->peerinfo;
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        ev_ctx.op = GD_FRIEND_UPDATE_ADD;
+
+        friends = dict_new ();
+        if (!friends)
+                goto out;
+
+        snprintf (key, sizeof (key), "op");
+        ret = dict_set_int32 (friends, key, ev_ctx.op);
+        if (ret)
+                goto out;
+
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
+                        continue;
+
+                count++;
+                snprintf (key, sizeof (key), "friend%d.uuid", count);
+                dup_buf = gf_strdup (uuid_utoa (peerinfo->uuid));
+                ret = dict_set_dynstr (friends, key, dup_buf);
+                if (ret)
+                        goto out;
+                snprintf (key, sizeof (key), "friend%d.hostname", count);
+                ret = dict_set_str (friends, key, peerinfo->hostname);
+                if (ret)
+                        goto out;
+                gf_log ("", GF_LOG_INFO, "Added uuid: %s, host: %s",
+                        dup_buf, peerinfo->hostname);
+        }
+
+        ret = dict_set_int32 (friends, "count", count);
+        if (ret)
+                goto out;
+
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                if (!peerinfo->connected || !peerinfo->peer)
+                        continue;
+
+                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
+                        continue;
+
+                ret = dict_set_static_ptr (friends, "peerinfo", peerinfo);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
+                        goto out;
+                }
+
+                proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
+                if (proc->fn) {
+                        ret = proc->fn (NULL, this, friends);
+                }
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+out:
+        if (friends)
+                dict_unref (friends);
+
+        return ret;
+}
+
+static int
+glusterd_peer_detach_cleanup (glusterd_conf_t *priv)
+{
+        int                     ret = -1;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_volinfo_t      *tmp_volinfo = NULL;
+
+        GF_ASSERT (priv);
+
+        list_for_each_entry_safe (volinfo,tmp_volinfo,
+                                  &priv->volumes, vol_list) {
+                if (!glusterd_friend_contains_vol_bricks (volinfo,
+                                                          MY_UUID)) {
+                        gf_log (THIS->name, GF_LOG_INFO,
+                                "Deleting stale volume %s", volinfo->volname);
+                        ret = glusterd_delete_volume (volinfo);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Error deleting stale volume");
+                                goto out;
+                        }
+                }
+        }
+        ret = 0;
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_ac_handle_friend_remove_req (glusterd_friend_sm_event_t *event,
+                                      void *ctx)
+{
+        int                             ret = 0;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_friend_req_ctx_t       *ev_ctx = NULL;
+        glusterd_friend_sm_event_t      *new_event = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        GF_ASSERT (ctx);
+        ev_ctx = ctx;
+        peerinfo = event->peerinfo;
+        GF_ASSERT (peerinfo);
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        ret = glusterd_xfer_friend_remove_resp (ev_ctx->req, ev_ctx->hostname,
+                                                ev_ctx->port);
+
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+
+                ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
+                                                    &new_event);
+                if (ret)
+                        goto out;
+
+                new_event->peerinfo = peerinfo;
+
+                ret = glusterd_friend_sm_inject_event (new_event);
+                if (ret)
+                        goto out;
+        }
+        ret = glusterd_peer_detach_cleanup (priv);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "Peer detach cleanup was not successful");
+                ret = 0;
+        }
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_ac_friend_remove (glusterd_friend_sm_event_t *event, void *ctx)
+{
+        int                     ret = -1;
+
+        ret = glusterd_friend_remove_cleanup_vols (event->peerinfo->uuid);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING, "Volumes cleanup failed");
+
+        ret = glusterd_friend_cleanup (event->peerinfo);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Cleanup returned: %d", ret);
+        }
+
+        return 0;
+}
+
+/*static int
+glusterd_ac_none (void *ctx)
+{
+        int ret = 0;
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}*/
+
+static int
+glusterd_ac_handle_friend_add_req (glusterd_friend_sm_event_t *event, void *ctx)
+{
+        int                             ret = 0;
+        uuid_t                          uuid;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_friend_req_ctx_t       *ev_ctx = NULL;
+        glusterd_friend_update_ctx_t    *new_ev_ctx = NULL;
+        glusterd_friend_sm_event_t      *new_event = NULL;
+        glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+        int                             status = 0;
+        int32_t                         op_ret = -1;
+        int32_t                         op_errno = 0;
+
+        GF_ASSERT (ctx);
+        ev_ctx = ctx;
+        uuid_copy (uuid, ev_ctx->uuid);
+        peerinfo = event->peerinfo;
+        GF_ASSERT (peerinfo);
+        uuid_copy (peerinfo->uuid, ev_ctx->uuid);
+
+        //Build comparison logic here.
+        ret = glusterd_compare_friend_data (ev_ctx->vols, &status,
+                                            peerinfo->hostname);
+        if (ret)
+                goto out;
+
+        if (GLUSTERD_VOL_COMP_RJT != status) {
+                event_type = GD_FRIEND_EVENT_LOCAL_ACC;
+                op_ret = 0;
+        } else {
+                event_type = GD_FRIEND_EVENT_LOCAL_RJT;
+                op_errno = GF_PROBE_VOLUME_CONFLICT;
+                op_ret = -1;
+        }
+
+        ret = glusterd_friend_sm_new_event (event_type, &new_event);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Out of Memory");
+        }
+
+        new_event->peerinfo = peerinfo;
+
+        new_ev_ctx = GF_CALLOC (1, sizeof (*new_ev_ctx),
+                                gf_gld_mt_friend_update_ctx_t);
+        if (!new_ev_ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (new_ev_ctx->uuid, ev_ctx->uuid);
+        new_ev_ctx->hostname = gf_strdup (ev_ctx->hostname);
+        new_ev_ctx->op = GD_FRIEND_UPDATE_ADD;
+
+        new_event->ctx = new_ev_ctx;
+
+        glusterd_friend_sm_inject_event (new_event);
+
+        ret = glusterd_xfer_friend_add_resp (ev_ctx->req, ev_ctx->hostname,
+                                             peerinfo->hostname, ev_ctx->port,
+                                             op_ret, op_errno);
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+static int
+glusterd_friend_sm_transition_state (glusterd_peerinfo_t *peerinfo,
+                                     glusterd_sm_t *state,
+                                     glusterd_friend_sm_event_type_t event_type)
+{
+
+        GF_ASSERT (state);
+        GF_ASSERT (peerinfo);
+
+        (void) glusterd_sm_tr_log_transition_add (&peerinfo->sm_log,
+                                           peerinfo->state.state,
+                                           state[event_type].next_state,
+                                           event_type);
+
+        peerinfo->state.state = state[event_type].next_state;
+        return 0;
+}
+
+
+glusterd_sm_t glusterd_state_default [] = {
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe},//EV_PROBE
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add}, //EV_INIT_FRIEND_REQ
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe}, //EVENT_CONNECTED
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_MAX
+};
+
+glusterd_sm_t glusterd_state_probe_rcvd [] = {
+        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},
+        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EV_PROBE
+        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EV_INIT_FRIEND_REQ
+        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_CONNECTED
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_MAX
+};
+
+glusterd_sm_t glusterd_state_connected_rcvd [] = {
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EV_PROBE
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EV_INIT_FRIEND_REQ
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_reverse_probe_begin}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_CONNECTED
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_MAX
+};
+
+glusterd_sm_t glusterd_state_connected_accepted [] = {
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_friend_probe}, //EV_PROBE
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_friend_add}, //EV_INIT_FRIEND_REQ
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_CONNECTED
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_MAX
+};
+
+glusterd_sm_t  glusterd_state_req_sent [] = {
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_NONE,
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_PROBE,
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
+        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_CONNECTED
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
+};
+
+glusterd_sm_t  glusterd_state_req_rcvd [] = {
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_NONE,
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_PROBE,
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},//EVENT_CONNECTED
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
+};
+
+glusterd_sm_t  glusterd_state_befriended [] = {
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_NONE,
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_PROBE,
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_friend_add},//EVENT_CONNECTED
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},//EVENT_MAX
+};
+
+glusterd_sm_t  glusterd_state_req_sent_rcvd [] = {
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_NONE,
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_PROBE,
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_CONNECTED
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_MAX
+};
+
+glusterd_sm_t  glusterd_state_rejected [] = {
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_NONE,
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_probe}, //EVENT_PROBE,
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add}, //EVENT_INIT_FRIEND_REQ,
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_add},//EVENT_CONNECTED
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
+};
+
+glusterd_sm_t  glusterd_state_req_accepted [] = {
+        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_NONE,
+        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_PROBE,
+        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_reverse_probe_begin},//EVENT_CONNECTED
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
+};
+
+glusterd_sm_t  glusterd_state_unfriend_sent [] = {
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_NONE,
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_PROBE,
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCV_FRIEND_REQ
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_INIT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_CONNECTED
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_MAX
+};
+
+glusterd_sm_t *glusterd_friend_state_table [] = {
+        glusterd_state_default,
+        glusterd_state_req_sent,
+        glusterd_state_req_rcvd,
+        glusterd_state_befriended,
+        glusterd_state_req_accepted,
+        glusterd_state_req_sent_rcvd,
+        glusterd_state_rejected,
+        glusterd_state_unfriend_sent,
+        glusterd_state_probe_rcvd,
+        glusterd_state_connected_rcvd,
+        glusterd_state_connected_accepted
+};
+
+int
+glusterd_friend_sm_new_event (glusterd_friend_sm_event_type_t event_type,
+                              glusterd_friend_sm_event_t **new_event)
+{
+        glusterd_friend_sm_event_t      *event = NULL;
+
+        GF_ASSERT (new_event);
+        GF_ASSERT (GD_FRIEND_EVENT_NONE <= event_type &&
+                        GD_FRIEND_EVENT_MAX > event_type);
+
+        event = GF_CALLOC (1, sizeof (*event), gf_gld_mt_friend_sm_event_t);
+
+        if (!event)
+                return -1;
+
+        *new_event = event;
+        event->event = event_type;
+        INIT_LIST_HEAD (&event->list);
+
+        return 0;
+}
+
+int
+glusterd_friend_sm_inject_event (glusterd_friend_sm_event_t *event)
+{
+        GF_ASSERT (event);
+        gf_log ("glusterd", GF_LOG_DEBUG, "Enqueue event: '%s'",
+                glusterd_friend_sm_event_name_get (event->event));
+        list_add_tail (&event->list, &gd_friend_sm_queue);
+
+        return 0;
+}
+
+void
+glusterd_destroy_friend_event_context (glusterd_friend_sm_event_t *event)
+{
+        if (!event)
+                return;
+
+        switch (event->event) {
+        case GD_FRIEND_EVENT_RCVD_FRIEND_REQ:
+        case GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND:
+                glusterd_destroy_friend_req_ctx (event->ctx);
+                break;
+        case GD_FRIEND_EVENT_LOCAL_ACC:
+        case GD_FRIEND_EVENT_LOCAL_RJT:
+        case GD_FRIEND_EVENT_RCVD_ACC:
+        case GD_FRIEND_EVENT_RCVD_RJT:
+                glusterd_destroy_friend_update_ctx (event->ctx);
+                break;
+        default:
+                break;
+        }
+}
+
+gf_boolean_t
+gd_does_peer_affect_quorum (glusterd_friend_sm_state_t old_state,
+                            glusterd_friend_sm_event_type_t event_type,
+                            glusterd_peerinfo_t *peerinfo)
+{
+        gf_boolean_t    affects = _gf_false;
+
+        //When glusterd comes up with friends in BEFRIENDED state in store,
+        //wait until compare-data happens.
+        if ((old_state == GD_FRIEND_STATE_BEFRIENDED) &&
+            (event_type != GD_FRIEND_EVENT_RCVD_ACC) &&
+            (event_type != GD_FRIEND_EVENT_LOCAL_ACC))
+                goto out;
+        if ((peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)
+            && peerinfo->connected) {
+                affects = _gf_true;
+        }
+out:
+        return affects;
+}
+
+int
+glusterd_friend_sm ()
+{
+        glusterd_friend_sm_event_t      *event      = NULL;
+        glusterd_friend_sm_event_t      *tmp        = NULL;
+        int                              ret        = -1;
+        glusterd_friend_sm_ac_fn         handler    = NULL;
+        glusterd_sm_t                   *state      = NULL;
+        glusterd_peerinfo_t             *peerinfo   = NULL;
+        glusterd_friend_sm_event_type_t  event_type = 0;
+        gf_boolean_t                     is_await_conn = _gf_false;
+        gf_boolean_t                     quorum_action = _gf_false;
+        glusterd_friend_sm_state_t       old_state = GD_FRIEND_STATE_DEFAULT;
+
+        while (!list_empty (&gd_friend_sm_queue)) {
+                list_for_each_entry_safe (event, tmp, &gd_friend_sm_queue, list) {
+
+                        list_del_init (&event->list);
+                        event_type = event->event;
+                        peerinfo = event->peerinfo;
+                        if (!peerinfo) {
+                                gf_log ("glusterd", GF_LOG_CRITICAL, "Received"
+                                        " event %s with empty peer info",
+                                glusterd_friend_sm_event_name_get (event_type));
+
+                                GF_FREE (event);
+                                continue;
+                        }
+                        gf_log ("", GF_LOG_DEBUG, "Dequeued event of type: '%s'",
+                                glusterd_friend_sm_event_name_get (event_type));
+
+
+                        old_state = peerinfo->state.state;
+                        state = glusterd_friend_state_table[peerinfo->state.state];
+
+                        GF_ASSERT (state);
+
+                        handler = state[event_type].handler;
+                        GF_ASSERT (handler);
+
+                        ret = handler (event, event->ctx);
+                        if (ret == GLUSTERD_CONNECTION_AWAITED) {
+                                is_await_conn = _gf_true;
+                                ret = 0;
+                        }
+
+                        if (ret) {
+                                gf_log ("glusterd", GF_LOG_ERROR, "handler returned: "
+                                                "%d", ret);
+                                glusterd_destroy_friend_event_context (event);
+                                GF_FREE (event);
+                                continue;
+                        }
+
+                        if ((GD_FRIEND_EVENT_REMOVE_FRIEND == event_type) ||
+                           (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND == event_type)){
+                                glusterd_destroy_friend_event_context (event);
+                                GF_FREE (event);
+                                continue;
+                        }
+
+                        ret = glusterd_friend_sm_transition_state (peerinfo,
+                                                            state, event_type);
+
+                        if (ret) {
+                                gf_log ("glusterd", GF_LOG_ERROR, "Unable to transition"
+                                        " state from '%s' to '%s' for event '%s'",
+                        glusterd_friend_sm_state_name_get(peerinfo->state.state),
+                        glusterd_friend_sm_state_name_get(state[event_type].next_state),
+                                glusterd_friend_sm_event_name_get(event_type));
+                                goto out;
+                        }
+
+                        if (gd_does_peer_affect_quorum (old_state, event_type,
+                                                        peerinfo)) {
+                                peerinfo->quorum_contrib = QUORUM_UP;
+                                if (peerinfo->quorum_action) {
+                                        peerinfo->quorum_action = _gf_false;
+                                        quorum_action = _gf_true;
+                                }
+                        }
+
+                        ret = glusterd_store_peerinfo (peerinfo);
+
+                        glusterd_destroy_friend_event_context (event);
+                        GF_FREE (event);
+                        if (is_await_conn)
+                                break;
+                }
+                if (is_await_conn)
+                        break;
+        }
+
+        ret = 0;
+out:
+        if (quorum_action) {
+            /* When glusterd is restarted, it needs to wait until the 'friends' view
+             * of the volumes settle, before it starts any of the internal daemons.
+             *
+             * Every friend that was part of the cluster, would send its
+             * cluster-view, 'our' way. For every friend, who belongs to
+             * a partition which has a different cluster-view from our
+             * partition, we may update our cluster-view. For subsequent
+             * friends from that partition would agree with us, if the first
+             * friend wasn't rejected. For every first friend, whom we agreed with,
+             * we would need to start internal daemons/bricks belonging to the
+             * new volumes.
+             * glusterd_spawn_daemons calls functions that are idempotent. ie,
+             * the functions spawn process(es) only if they are not started yet.
+             *
+             * */
+                glusterd_spawn_daemons (NULL);
+                glusterd_do_quorum_action ();
+        }
+        return ret;
+}
+
+
+int
+glusterd_friend_sm_init ()
+{
+        INIT_LIST_HEAD (&gd_friend_sm_queue);
+        return 0;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
new file mode 100644
index 000000000..b9bedbe69
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -0,0 +1,217 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_SM_H_
+#define _GLUSTERD_SM_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <pthread.h>
+#include "uuid.h"
+
+#include "rpc-clnt.h"
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "call-stub.h"
+#include "fd.h"
+#include "byte-order.h"
+//#include "glusterd.h"
+#include "rpcsvc.h"
+#include "store.h"
+
+typedef enum gd_quorum_contribution_ {
+        QUORUM_NONE,
+        QUORUM_WAITING,
+        QUORUM_DOWN,
+        QUORUM_UP
+} gd_quorum_contrib_t;
+
+typedef enum gd_quorum_status_ {
+        QUORUM_UNKNOWN,
+        QUORUM_NOT_APPLICABLE,
+        QUORUM_MEETS,
+        QUORUM_DOES_NOT_MEET
+} gd_quorum_status_t;
+
+typedef enum glusterd_friend_sm_state_ {
+        GD_FRIEND_STATE_DEFAULT = 0,
+        GD_FRIEND_STATE_REQ_SENT,
+        GD_FRIEND_STATE_REQ_RCVD,
+        GD_FRIEND_STATE_BEFRIENDED,
+        GD_FRIEND_STATE_REQ_ACCEPTED,
+        GD_FRIEND_STATE_REQ_SENT_RCVD,
+        GD_FRIEND_STATE_REJECTED,
+        GD_FRIEND_STATE_UNFRIEND_SENT,
+        GD_FRIEND_STATE_PROBE_RCVD,
+        GD_FRIEND_STATE_CONNECTED_RCVD,
+        GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+        GD_FRIEND_STATE_MAX
+} glusterd_friend_sm_state_t;
+
+typedef struct glusterd_peer_state_info_ {
+        glusterd_friend_sm_state_t   state;
+        struct timeval          transition_time;
+}glusterd_peer_state_info_t;
+
+typedef struct glusterd_peer_hostname_ {
+        char                    *hostname;
+        struct list_head        hostname_list;
+}glusterd_peer_hostname_t;
+
+typedef struct glusterd_sm_transition_ {
+        int             old_state;
+        int             event;
+        int             new_state;
+        time_t          time;
+} glusterd_sm_transition_t;
+
+typedef struct glusterd_sm_tr_log_ {
+        glusterd_sm_transition_t    *transitions;
+        size_t                      current;
+        size_t                      size;
+        size_t                      count;
+        char*                       (*state_name_get) (int);
+        char*                       (*event_name_get) (int);
+} glusterd_sm_tr_log_t;
+
+struct glusterd_peerinfo_ {
+        uuid_t                          uuid;
+        char                            uuid_str[50]; /* Retrieve this using
+                                                       * gd_peer_uuid_str ()
+                                                       */
+        glusterd_peer_state_info_t      state;
+        char                            *hostname;
+        int                             port;
+        struct list_head                uuid_list;
+        struct list_head                op_peers_list;
+        struct rpc_clnt                 *rpc;
+        rpc_clnt_prog_t                 *mgmt;
+        rpc_clnt_prog_t                 *peer;
+        rpc_clnt_prog_t                 *mgmt_v3;
+        int                             connected;
+        gf_store_handle_t         *shandle;
+        glusterd_sm_tr_log_t            sm_log;
+        gf_boolean_t                    quorum_action;
+        gd_quorum_contrib_t             quorum_contrib;
+        gf_boolean_t                    locked;
+};
+
+typedef struct glusterd_peerinfo_ glusterd_peerinfo_t;
+
+typedef enum glusterd_ev_gen_mode_ {
+        GD_MODE_OFF,
+        GD_MODE_ON,
+        GD_MODE_SWITCH_ON
+} glusterd_ev_gen_mode_t;
+
+typedef struct glusterd_peer_ctx_args_ {
+        rpcsvc_request_t        *req;
+        glusterd_ev_gen_mode_t  mode;
+        dict_t                  *dict;
+} glusterd_peerctx_args_t;
+
+typedef struct glusterd_peer_ctx_ {
+        glusterd_peerctx_args_t        args;
+        glusterd_peerinfo_t            *peerinfo;
+        char                           *errstr;
+} glusterd_peerctx_t;
+
+typedef enum glusterd_friend_sm_event_type_ {
+        GD_FRIEND_EVENT_NONE = 0,
+        GD_FRIEND_EVENT_PROBE,
+        GD_FRIEND_EVENT_INIT_FRIEND_REQ,
+        GD_FRIEND_EVENT_RCVD_ACC,
+        GD_FRIEND_EVENT_LOCAL_ACC,
+        GD_FRIEND_EVENT_RCVD_RJT,
+        GD_FRIEND_EVENT_LOCAL_RJT,
+        GD_FRIEND_EVENT_RCVD_FRIEND_REQ,
+        GD_FRIEND_EVENT_INIT_REMOVE_FRIEND,
+        GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND,
+        GD_FRIEND_EVENT_REMOVE_FRIEND,
+        GD_FRIEND_EVENT_CONNECTED,
+        GD_FRIEND_EVENT_MAX
+} glusterd_friend_sm_event_type_t;
+
+
+typedef enum glusterd_friend_update_op_ {
+        GD_FRIEND_UPDATE_NONE = 0,
+        GD_FRIEND_UPDATE_ADD,
+        GD_FRIEND_UPDATE_DEL,
+} glusterd_friend_update_op_t;
+
+
+struct glusterd_friend_sm_event_ {
+        struct list_head        list;
+        glusterd_peerinfo_t     *peerinfo;
+        void                    *ctx;
+        glusterd_friend_sm_event_type_t event;
+};
+
+typedef struct glusterd_friend_sm_event_ glusterd_friend_sm_event_t;
+
+typedef int (*glusterd_friend_sm_ac_fn) (glusterd_friend_sm_event_t *, void *);
+
+typedef struct glusterd_sm_ {
+        glusterd_friend_sm_state_t      next_state;
+        glusterd_friend_sm_ac_fn        handler;
+} glusterd_sm_t;
+
+typedef struct glusterd_friend_req_ctx_ {
+        uuid_t                  uuid;
+        char                    *hostname;
+        rpcsvc_request_t        *req;
+        int                      port;
+        dict_t                  *vols;
+} glusterd_friend_req_ctx_t;
+
+typedef struct glusterd_friend_update_ctx_ {
+        uuid_t                  uuid;
+        char                    *hostname;
+        int                     op;
+} glusterd_friend_update_ctx_t;
+
+typedef struct glusterd_probe_ctx_ {
+        char                    *hostname;
+        rpcsvc_request_t        *req;
+        int                      port;
+        dict_t                  *dict;
+} glusterd_probe_ctx_t;
+int
+glusterd_friend_sm_new_event (glusterd_friend_sm_event_type_t event_type,
+                              glusterd_friend_sm_event_t **new_event);
+int
+glusterd_friend_sm_inject_event (glusterd_friend_sm_event_t *event);
+
+int
+glusterd_friend_sm_init ();
+
+int
+glusterd_friend_sm ();
+
+void
+glusterd_destroy_probe_ctx (glusterd_probe_ctx_t *ctx);
+
+void
+glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx);
+
+char*
+glusterd_friend_sm_state_name_get (int state);
+
+char*
+glusterd_friend_sm_event_name_get (int event);
+
+int
+glusterd_broadcast_friend_delete (char *hostname, uuid_t uuid);
+void
+glusterd_destroy_friend_update_ctx (glusterd_friend_update_ctx_t *ctx);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
new file mode 100644
index 000000000..9b811cd05
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -0,0 +1,5590 @@
+/*
+   Copyright (c) 2013-2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <inttypes.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/resource.h>
+#include <sys/statvfs.h>
+#include <sys/mount.h>
+
+#include "globals.h"
+#include "compat.h"
+#include "protocol-common.h"
+#include "xlator.h"
+#include "logging.h"
+#include "timer.h"
+#include "glusterd-mem-types.h"
+#include "glusterd.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "run.h"
+#include "glusterd-volgen.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-syncop.h"
+
+#include "syscall.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+
+#ifdef GF_LINUX_HOST_OS
+#include <mntent.h>
+#endif
+
+char snap_mount_folder[PATH_MAX];
+
+static int32_t
+glusterd_find_missed_snap (dict_t *rsp_dict, glusterd_volinfo_t *vol,
+                           char *snap_uuid, struct list_head *peers,
+                           int32_t op);
+
+/* This function will restore a snapshot volumes
+ *
+ * @param dict          dictionary containing snapshot restore request
+ * @param op_errstr     In case of any failure error message will be returned
+ *                      in this variable
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_snapshot_restore (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int                     ret             = -1;
+        char                    *volname        = NULL;
+        char                    *snapname       = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_volinfo_t      *snap_volinfo   = NULL;
+        glusterd_volinfo_t      *volinfo        = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        glusterd_conf_t         *priv           = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                        "snap name");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (NULL == snap) {
+                ret = gf_asprintf (op_errstr, "Snap (%s) not found",
+                                        snapname);
+                if (ret < 0) {
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+
+        /* TODO : As of now there is only volume in snapshot.
+         * Change this when multiple volume snapshot is introduced
+         */
+        snap_volinfo = list_entry (snap->volumes.next, glusterd_volinfo_t,
+                                   vol_list);
+
+        ret = glusterd_volinfo_find (snap_volinfo->parent_volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not get volinfo of "
+                        "%s", snap_volinfo->parent_volname);
+                goto out;
+        }
+
+        if (is_origin_glusterd (dict) == _gf_true) {
+                /* From origin glusterd check if      *
+                 * any peers with snap bricks is down */
+                ret = glusterd_find_missed_snap (rsp_dict, snap_volinfo,
+                                                 snap_volinfo->volname,
+                                                 &priv->peers,
+                                                 GF_SNAP_OPTION_TYPE_RESTORE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to find missed snap restores");
+                        goto out;
+                }
+        }
+
+        ret = gd_restore_snap_volume (rsp_dict, volinfo, snap_volinfo);
+        if (ret) {
+                /* No need to update op_errstr because it is assumed
+                 * that the called function will do that in case of
+                 * failure.
+                 */
+                gf_log (this->name, GF_LOG_ERROR, "Failed to restore "
+                        "snap for %s volume", volname);
+                goto out;
+        }
+
+        ret = 0;
+
+        /* TODO: Need to check if we need to delete the snap after the
+         * operation is successful or not. Also need to persist the state
+         * of restore operation in the store.
+         */
+out:
+        return ret;
+}
+
+/* This function is called before actual restore is taken place. This function
+ * will validate whether the snapshot volumes are ready to be restored or not.
+ *
+ * @param dict          dictionary containing snapshot restore request
+ * @param op_errstr     In case of any failure error message will be returned
+ *                      in this variable
+ * @param rsp_dict      response dictionary
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_snapshot_restore_prevalidate (dict_t *dict, char **op_errstr,
+                                       dict_t *rsp_dict)
+{
+        int                     ret             = -1;
+        int32_t                 i               = 0;
+        int32_t                 volcount        = 0;
+        gf_boolean_t            snap_restored   = _gf_false;
+        char                    key[PATH_MAX]   = {0, };
+        char                    *volname        = NULL;
+        char                    *snapname       = NULL;
+        glusterd_volinfo_t      *volinfo        = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        xlator_t                *this           = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                        "snap name");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (NULL == snap) {
+                ret = gf_asprintf (op_errstr, "Snap (%s) not found",
+                                snapname);
+                if (ret < 0) {
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+
+        snap_restored = snap->snap_restored;
+
+        if (snap_restored) {
+                ret = gf_asprintf (op_errstr, "Snap (%s) is already "
+                                  "restored", snapname);
+                if (ret < 0) {
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (rsp_dict, "snapname", snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "snap name");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "volcount", &volcount);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume count");
+                goto out;
+        }
+
+        /* Snapshot restore will only work if all the volumes,
+           that are part of the snapshot, are stopped. */
+        for (i = 1; i <= volcount; ++i) {
+                snprintf (key, sizeof (key), "volname%d", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "get volume name");
+                        goto out;
+                }
+
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        ret = gf_asprintf (op_errstr, "Volume (%s) not found",
+                                           volname);
+                        if (ret < 0) {
+                                goto out;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (glusterd_is_volume_started (volinfo)) {
+                        ret = gf_asprintf (op_errstr, "Volume (%s) has been "
+                        "started. Volume needs to be stopped before restoring "
+                        "a snapshot.", volname);
+                        if (ret < 0) {
+                                goto out;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+snap_max_hard_limits_validate (dict_t *dict, char *volname,
+                          uint64_t value, char **op_errstr)
+{
+        char                err_str[PATH_MAX] = "";
+        glusterd_conf_t    *conf              = NULL;
+        glusterd_volinfo_t *volinfo           = NULL;
+        int                 ret               = -1;
+        uint64_t            max_limit         = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+        xlator_t           *this              = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        if (volname) {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (!ret) {
+                        if (volinfo->is_snap_volume) {
+                                ret = -1;
+                                snprintf (err_str, PATH_MAX,
+                                         "%s is a snap volume. Configuring "
+                                         "snap-max-hard-limit for a snap "
+                                         "volume is prohibited.", volname);
+                                goto out;
+                        }
+                }
+        }
+
+        if (value) {
+                /* Max limit for the system is GLUSTERD_SNAPS_MAX_HARD_LIMIT
+                 * but max limit for a volume is conf->snap_max_hard_limit.
+                 */
+                if (volname) {
+                        max_limit = conf->snap_max_hard_limit;
+                } else {
+                        max_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+                }
+        }
+
+        if ((value < 0) || (value > max_limit)) {
+                ret = -1;
+                snprintf (err_str, PATH_MAX, "Invalid snap-max-hard-limit"
+                          "%"PRIu64 ". Expected range 0 - %"PRIu64,
+                          value, max_limit);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                *op_errstr = gf_strdup (err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+        }
+        return ret;
+}
+
+int
+glusterd_snapshot_config_prevalidate (dict_t *dict, char **op_errstr)
+{
+        char               *volname             = NULL;
+        glusterd_volinfo_t *volinfo             = NULL;
+        xlator_t           *this                = NULL;
+        int                 ret                 = -1;
+        int                 config_command      = 0;
+        char                err_str[PATH_MAX]   = {0,};
+        glusterd_conf_t    *conf                = NULL;
+        uint64_t            value               = 0;
+        uint64_t            hard_limit          = 0;
+        uint64_t            soft_limit          = 0;
+        gf_loglevel_t       loglevel            = GF_LOG_ERROR;
+        uint64_t            max_limit           = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        ret = dict_get_int32 (dict, "config-command", &config_command);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str),
+                          "failed to get config-command type");
+                goto out;
+        }
+
+        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
+
+        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (volname) {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "Volume %s does not exist.", volname);
+                        goto out;
+                }
+        }
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+                if (hard_limit) {
+                        /* Validations for snap-max-hard-limits */
+                        ret = snap_max_hard_limits_validate (dict, volname,
+                                                         hard_limit, op_errstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "snap-max-hard-limit validation "
+                                        "failed.");
+                                goto out;
+                        }
+                }
+
+                if (soft_limit) {
+                        max_limit = GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT;
+                        if ((soft_limit < 0) || (soft_limit > max_limit)) {
+                                ret = -1;
+                                snprintf (err_str, PATH_MAX, "Invalid "
+                                         "snap-max-soft-limit ""%"
+                                         PRIu64 ". Expected range 0 - %"PRIu64,
+                                         value, max_limit);
+                                goto out;
+                        }
+                        break;
+                }
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+
+        if (ret && err_str[0] != '\0') {
+                gf_log (this->name, loglevel, "%s", err_str);
+                *op_errstr = gf_strdup (err_str);
+        }
+
+        return ret;
+}
+
+int
+glusterd_snap_create_pre_val_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char                  *snap_brick_dir        = NULL;
+        char                  *snap_device           = NULL;
+        char                  *tmpstr                = NULL;
+        char                   key[PATH_MAX]         = "";
+        char                   snapbrckcnt[PATH_MAX] = "";
+        char                   snapbrckord[PATH_MAX] = "";
+        int                    ret                   = -1;
+        int64_t                i                     = -1;
+        int64_t                j                     = -1;
+        int64_t                volume_count          = 0;
+        int64_t                brick_count           = 0;
+        int64_t                brick_order           = 0;
+        xlator_t              *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dst);
+        GF_ASSERT (src);
+
+        ret = dict_get_int64 (src, "volcount", &volume_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                        "get the volume count");
+                goto out;
+        }
+
+        for (i = 0; i < volume_count; i++) {
+                memset (snapbrckcnt, '\0', sizeof(snapbrckcnt));
+                ret = snprintf (snapbrckcnt, sizeof(snapbrckcnt) - 1,
+                                "vol%ld_brickcount", i+1);
+                ret = dict_get_int64 (src, snapbrckcnt, &brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "No bricks for this volume in this dict");
+                        continue;
+                }
+
+                for (j = 0; j < brick_count; j++) {
+                        /* Fetching data from source dict */
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brickdir%ld", i+1, j);
+
+                        ret = dict_get_ptr (src, key,
+                                            (void **)&snap_brick_dir);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Unable to fetch %s", key);
+                                continue;
+                        }
+
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brick_snapdevice%ld", i+1, j);
+
+                        ret = dict_get_ptr (src, key,
+                                            (void **)&snap_device);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to fetch snap_device");
+                                goto out;
+                        }
+
+                        snprintf (snapbrckord, sizeof(snapbrckord) - 1,
+                                  "vol%ld.brick%ld.order", i+1, j);
+
+                        ret = dict_get_int64 (src, snapbrckord, &brick_order);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get brick order");
+                                goto out;
+                        }
+
+                        /* Adding the data in the dst dict */
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brickdir%ld", i+1, brick_order);
+
+                        tmpstr = gf_strdup (snap_brick_dir);
+                        if (!tmpstr) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Out Of Memory");
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = dict_set_dynstr (dst, key, tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                GF_FREE (tmpstr);
+                                goto out;
+                        }
+
+                        snprintf (key, sizeof(key) - 1,
+                                  "vol%ld.brick_snapdevice%ld",
+                                  i+1, brick_order);
+
+                        tmpstr = gf_strdup (snap_device);
+                        if (!tmpstr) {
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = dict_set_dynstr (dst, key, tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                GF_FREE (tmpstr);
+                                goto out;
+                        }
+
+                }
+        }
+
+        ret = 0;
+out:
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        int             ret             = -1;
+        int32_t         snap_command    = 0;
+        xlator_t       *this            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dst || !src) {
+                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = glusterd_snap_create_pre_val_use_rsp_dict (dst, src);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to use "
+                                "rsp dict");
+                        goto out;
+                }
+                break;
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_snapshot_create_prevalidate (dict_t *dict, char **op_errstr,
+                                      dict_t *rsp_dict)
+{
+        char                  *volname                   = NULL;
+        char                  *snapname                  = NULL;
+        char                  *device                    = NULL;
+        char                  *tmpstr                    = NULL;
+        char                  *brick_dir                 = NULL;
+        char                   snap_brick_dir[PATH_MAX]  = "";
+        char                  *mnt_pt                    = NULL;
+        char                   key[PATH_MAX]             = "";
+        char                   snap_mount[PATH_MAX]      = "";
+        char                   snap_volname[64]          = "";
+        char                   err_str[PATH_MAX]         = "";
+        int                    ret                       = -1;
+        int64_t                i                         = 0;
+        int64_t                volcount                  = 0;
+        int64_t                brick_count               = 0;
+        int64_t                brick_order               = 0;
+        glusterd_brickinfo_t  *brickinfo                 = NULL;
+        glusterd_volinfo_t    *volinfo                   = NULL;
+        xlator_t              *this                      = NULL;
+        uuid_t                *snap_volid                = NULL;
+        gf_loglevel_t          loglevel                  = GF_LOG_ERROR;
+        glusterd_conf_t       *conf                      = NULL;
+        int64_t                effective_max_limit       = 0;
+
+        this = THIS;
+        GF_ASSERT (op_errstr);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = dict_get_int64 (dict, "volcount", &volcount);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to "
+                          "get the volume count");
+                goto out;
+        }
+        if (volcount <= 0) {
+                snprintf (err_str, sizeof (err_str), "Invalid volume count %ld "
+                          "supplied", volcount);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to get snapname");
+                goto out;
+        }
+
+        if (glusterd_find_snap_by_name (snapname)) {
+                ret = -1;
+                snprintf (err_str, sizeof (err_str), "Snap %s already exists",
+                          snapname);
+                goto out;
+        }
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%ld", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "failed to get volume name");
+                        goto out;
+                }
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "Volume (%s) does not exist ", volname);
+                        goto out;
+                }
+
+                ret = -1;
+                if (!glusterd_is_volume_started (volinfo)) {
+                        snprintf (err_str, sizeof (err_str), "volume %s is "
+                                  "not started", volinfo->volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+                if (glusterd_is_defrag_on (volinfo)) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "rebalance process is running for the "
+                                  "volume %s", volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+                /* TODO: Also check whether geo replication is running */
+
+                if (volinfo->is_snap_volume == _gf_true) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "Volume %s is a snap volume", volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+
+                if (volinfo->snap_max_hard_limit < conf->snap_max_hard_limit)
+                        effective_max_limit = volinfo->snap_max_hard_limit;
+                else
+                        effective_max_limit = conf->snap_max_hard_limit;
+
+                if (volinfo->snap_count >= effective_max_limit) {
+                        snprintf (err_str, sizeof (err_str),
+                                  "The number of existing snaps has reached "
+                                  "the effective maximum limit of %"PRIu64" ,"
+                                  "for the volume %s", effective_max_limit,
+                                  volname);
+                        loglevel = GF_LOG_WARNING;
+                        goto out;
+                }
+
+                snprintf (key, sizeof(key) - 1, "vol%ld_volid", i);
+                ret = dict_get_bin (dict, key, (void **)&snap_volid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch snap_volid");
+                        goto out;
+                }
+
+                /* snap volume uuid is used as lvm snapshot name.
+                   This will avoid restrictions on snapshot names
+                   provided by user */
+                GLUSTERD_GET_UUID_NOHYPHEN (snap_volname, *snap_volid);
+
+                brick_count = 0;
+                brick_order = 0;
+                /* Adding snap bricks mount paths to the dict */
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
+                                brick_order++;
+                                continue;
+                        }
+
+                        if (!glusterd_is_brick_started (brickinfo)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "brick %s:%s is not started",
+                                        brickinfo->hostname,
+                                        brickinfo->path);
+                                brick_order++;
+                                brick_count++;
+                                continue;
+                        }
+
+                        device = glusterd_get_brick_mount_details (brickinfo);
+                        if (!device) {
+                                snprintf (err_str, sizeof (err_str),
+                                          "getting device name for the brick "
+                                          "%s:%s failed", brickinfo->hostname,
+                                          brickinfo->path);
+                                ret = -1;
+                                goto out;
+                        }
+
+                        device = glusterd_build_snap_device_path (device,
+                                                                  snap_volname);
+                        if (!device) {
+                                snprintf (err_str, sizeof (err_str),
+                                          "cannot copy the snapshot device "
+                                          "name (volname: %s, snapname: %s)",
+                                          volinfo->volname, snapname);
+                                loglevel = GF_LOG_WARNING;
+                                ret = -1;
+                                goto out;
+                        }
+
+                        snprintf (key, sizeof(key),
+                                  "vol%ld.brick_snapdevice%ld", i,
+                                  brick_count);
+                        ret = dict_set_dynstr (rsp_dict, key, device);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                GF_FREE (device);
+                                goto out;
+                        }
+
+                        ret = glusterd_get_brick_root (brickinfo->path,
+                                                       &mnt_pt);
+                        if (ret) {
+                                snprintf (err_str, sizeof (err_str),
+                                "could not get the root of the brick path %s",
+                                brickinfo->path);
+                                loglevel = GF_LOG_WARNING;
+                                goto out;
+                        }
+                        if (strncmp (brickinfo->path, mnt_pt, strlen(mnt_pt))) {
+                                snprintf (err_str, sizeof (err_str),
+                                "brick: %s brick mount: %s",
+                                 brickinfo->path, mnt_pt);
+                                loglevel = GF_LOG_WARNING;
+                                goto out;
+                        }
+
+                        brick_dir = &brickinfo->path[strlen (mnt_pt)];
+                        brick_dir++;
+
+                        snprintf (snap_brick_dir, sizeof (snap_brick_dir),
+                                  "/%s", brick_dir);
+
+                        tmpstr = gf_strdup (snap_brick_dir);
+                        if (!tmpstr) {
+                                ret = -1;
+                                goto out;
+                        }
+                        snprintf (key, sizeof(key), "vol%ld.brickdir%ld", i,
+                                  brick_count);
+                        ret = dict_set_dynstr (rsp_dict, key, tmpstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", snap_mount);
+                                goto out;
+                        }
+                        tmpstr = NULL;
+
+                        snprintf (key, sizeof(key) - 1, "vol%ld.brick%ld.order",
+                                  i, brick_count);
+                        ret = dict_set_int64 (rsp_dict, key, brick_order);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set %s", key);
+                                goto out;
+                        }
+
+                        brick_count++;
+                        brick_order++;
+                }
+                snprintf (key, sizeof(key) - 1, "vol%ld_brickcount", i);
+                ret = dict_set_int64 (rsp_dict, key, brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
+                                key);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                GF_FREE (tmpstr);
+
+        if (ret && err_str[0] != '\0') {
+                gf_log (this->name, loglevel, "%s", err_str);
+                *op_errstr = gf_strdup (err_str);
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+glusterd_snap_t*
+glusterd_new_snap_object()
+{
+        glusterd_snap_t        *snap    = NULL;
+
+        snap = GF_CALLOC (1, sizeof (*snap), gf_gld_mt_snap_t);
+
+        if (snap) {
+                if (LOCK_INIT (&snap->lock)) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "Failed initiating"
+                                " snap lock");
+                        GF_FREE (snap);
+                        return NULL;
+                }
+
+                INIT_LIST_HEAD (&snap->snap_list);
+                INIT_LIST_HEAD (&snap->volumes);
+                snap->snapname[0] = 0;
+                snap->snap_status = GD_SNAP_STATUS_INIT;
+        }
+
+        return snap;
+
+};
+
+/* Function glusterd_list_add_snapvol adds the volinfo object (snapshot volume)
+   to the snapshot object list and to the parent volume list */
+int32_t
+glusterd_list_add_snapvol (glusterd_volinfo_t *origin_vol,
+                           glusterd_volinfo_t *snap_vol)
+{
+        int ret               = -1;
+        glusterd_snap_t *snap = NULL;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", origin_vol, out);
+        GF_VALIDATE_OR_GOTO ("glusterd", snap_vol, out);
+
+        snap = snap_vol->snapshot;
+        GF_ASSERT (snap);
+
+        list_add_tail (&snap_vol->vol_list, &snap->volumes);
+        LOCK (&origin_vol->lock);
+        {
+                list_add_order (&snap_vol->snapvol_list,
+                               &origin_vol->snap_volumes,
+                               glusterd_compare_snap_vol_time);
+                origin_vol->snap_count++;
+        }
+        UNLOCK (&origin_vol->lock);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Snap %s added to the list",
+                snap->snapname);
+        ret = 0;
+out:
+        return ret;
+}
+
+glusterd_snap_t*
+glusterd_find_snap_by_name (char *snapname)
+{
+        glusterd_snap_t *snap  = NULL;
+        glusterd_conf_t *priv  = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snapname);
+
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (!strcmp (snap->snapname, snapname)) {
+                        gf_log (THIS->name, GF_LOG_DEBUG, "Found "
+                                "snap %s (%s)", snap->snapname,
+                                uuid_utoa (snap->snap_id));
+                        goto out;
+                }
+        }
+        snap = NULL;
+out:
+        return snap;
+}
+
+glusterd_snap_t*
+glusterd_find_snap_by_id (uuid_t snap_id)
+{
+        glusterd_snap_t *snap  = NULL;
+        glusterd_conf_t *priv  = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        if (uuid_is_null(snap_id))
+                goto out;
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (!uuid_compare (snap->snap_id, snap_id)) {
+                        gf_log (THIS->name, GF_LOG_DEBUG, "Found "
+                                "snap %s (%s)", snap->snapname,
+                                uuid_utoa (snap->snap_id));
+                        goto out;
+                }
+        }
+        snap = NULL;
+out:
+        return snap;
+}
+
+int
+glusterd_do_lvm_snapshot_remove (glusterd_volinfo_t *snap_vol,
+                                 glusterd_brickinfo_t *brickinfo,
+                                 const char *mount_pt, const char *snap_device)
+{
+        int                     ret               = -1;
+        xlator_t               *this              = NULL;
+        glusterd_conf_t        *priv              = NULL;
+        runner_t                runner            = {0,};
+        char                    msg[1024]         = {0, };
+        char                    pidfile[PATH_MAX] = {0, };
+        pid_t                   pid               = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!brickinfo) {
+                gf_log (this->name, GF_LOG_ERROR, "brickinfo NULL");
+                goto out;
+        }
+
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (mount_pt);
+        GF_ASSERT (snap_device);
+
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_vol, brickinfo, priv);
+        if (glusterd_is_service_running (pidfile, &pid)) {
+                ret = kill (pid, SIGKILL);
+                if (ret && errno != ESRCH) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to kill pid "
+                                "%d reason : %s", pid, strerror(errno));
+                        goto out;
+                }
+        }
+
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "umount the snapshot mounted path %s",
+                  mount_pt);
+        runner_add_args (&runner, "umount", mount_pt, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+
+        /* We need not do synclock_unlock => runner_run => synclock_lock here.
+           Because it is needed if we are running a glusterfs process in
+           runner_run, so that when the glusterfs process started wants to
+           communicate to glusterd, glusterd wont be able to respond if it
+           has held the big lock. So we do unlock, run glusterfs process
+           (thus communicate to glusterd), lock. But since this is not a
+           glusterfs command that is being run, unlocking and then relocking
+           is not needed.
+        */
+        ret = runner_run (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "unmounting the "
+                        "path %s (brick: %s) failed (%s)", mount_pt,
+                        brickinfo->path, strerror (errno));
+                goto out;
+        }
+
+        runinit (&runner);
+        snprintf (msg, sizeof(msg), "remove snapshot of the brick %s:%s, "
+                  "device: %s", brickinfo->hostname, brickinfo->path,
+                  snap_device);
+        runner_add_args (&runner, "/sbin/lvremove", "-f", snap_device, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+
+        ret = runner_run (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "removing snapshot of the "
+                        "brick (%s:%s) of device %s failed",
+                        brickinfo->hostname, brickinfo->path, snap_device);
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
+{
+        char                 *mnt_pt        = NULL;
+        struct mntent        *entry         = NULL;
+        int32_t               brick_count   = -1;
+        int32_t               ret           = -1;
+        glusterd_brickinfo_t *brickinfo     = NULL;
+        xlator_t             *this          = NULL;
+        FILE                 *mtab          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_vol);
+
+        if (!snap_vol) {
+                gf_log (this->name, GF_LOG_ERROR, "snap volinfo is NULL");
+                goto out;
+        }
+
+        brick_count = -1;
+        list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                brick_count++;
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                if (brickinfo->snap_status == -1) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "snapshot was pending. lvm not present "
+                                "for brick %s:%s of the snap %s.",
+                                brickinfo->hostname, brickinfo->path,
+                                snap_vol->snapshot->snapname);
+
+                        /* Adding missed delete to the dict */
+                        ret = glusterd_add_missed_snaps_to_dict
+                                                   (rsp_dict,
+                                                    snap_vol->volname,
+                                                    brickinfo,
+                                                    brick_count + 1,
+                                                    GF_SNAP_OPTION_TYPE_DELETE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to add missed snapshot info "
+                                        "for %s:%s in the rsp_dict",
+                                        brickinfo->hostname,
+                                        brickinfo->path);
+                                goto out;
+                        }
+
+                        continue;
+                }
+
+                ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "getting the root "
+                               "of the brick for volume %s (snap %s) failed ",
+                               snap_vol->volname, snap_vol->snapshot->snapname);
+                        goto out;
+                }
+
+                entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+                if (!entry) {
+                        gf_log (this->name, GF_LOG_WARNING, "getting the mount"
+                                " entry for the brick %s:%s of the snap %s "
+                                "(volume: %s) failed", brickinfo->hostname,
+                                brickinfo->path, snap_vol->snapshot->snapname,
+                                snap_vol->volname);
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_do_lvm_snapshot_remove (snap_vol, brickinfo,
+                                                       mnt_pt,
+                                                       entry->mnt_fsname);
+                if (mtab)
+                        endmntent (mtab);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "remove the snapshot %s (%s)",
+                                brickinfo->path, entry->mnt_fsname);
+                        goto out;
+                }
+
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snap_volume_remove (dict_t *rsp_dict,
+                             glusterd_volinfo_t *snap_vol,
+                             gf_boolean_t remove_lvm,
+                             gf_boolean_t force)
+{
+        int                   ret         = -1;
+        int                   save_ret    =  0;
+        glusterd_brickinfo_t *brickinfo   = NULL;
+        glusterd_volinfo_t   *origin_vol  = NULL;
+        xlator_t             *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_vol);
+
+        if (!snap_vol) {
+                gf_log(this->name, GF_LOG_WARNING, "snap_vol in NULL");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                ret = glusterd_brick_stop (snap_vol, brickinfo, _gf_false);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING, "Failed to stop "
+                               "brick for volume %s", snap_vol->volname);
+                        save_ret = ret;
+
+                        /* Continue to cleaning up the snap in case of error
+                           if force flag is enabled */
+                        if (!force)
+                                goto out;
+                }
+        }
+
+        /* Only remove the backend lvm when required */
+        if (remove_lvm) {
+                ret = glusterd_lvm_snapshot_remove (rsp_dict, snap_vol);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING, "Failed to remove "
+                               "lvm snapshot volume %s", snap_vol->volname);
+                        save_ret = ret;
+                        if (!force)
+                                goto out;
+                }
+        }
+
+        ret = glusterd_store_delete_volume (snap_vol);
+        if (ret) {
+                gf_log(this->name, GF_LOG_WARNING, "Failed to remove volume %s "
+                       "from store", snap_vol->volname);
+                save_ret = ret;
+                if (!force)
+                        goto out;
+        }
+
+        if (!list_empty(&snap_vol->snapvol_list)) {
+                ret = glusterd_volinfo_find (snap_vol->parent_volname,
+                                             &origin_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "parent volinfo %s  for volume  %s",
+                                snap_vol->parent_volname, snap_vol->volname);
+                        save_ret = ret;
+                        if (!force)
+                                goto out;
+                }
+                origin_vol->snap_count--;
+        }
+
+        ret = glusterd_volinfo_delete (snap_vol);
+        if (ret) {
+                gf_log(this->name, GF_LOG_WARNING, "Failed to remove volinfo "
+                       "%s ", snap_vol->volname);
+                save_ret = ret;
+                if (!force)
+                        goto out;
+        }
+
+        if (save_ret)
+                ret = save_ret;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snapobject_delete (glusterd_snap_t *snap)
+{
+        if (snap == NULL) {
+                gf_log(THIS->name, GF_LOG_WARNING, "snap is NULL");
+                return -1;
+        }
+
+        list_del_init (&snap->snap_list);
+        list_del_init (&snap->volumes);
+        if (LOCK_DESTROY(&snap->lock))
+                gf_log (THIS->name, GF_LOG_WARNING, "Failed destroying lock"
+                        "of snap %s", snap->snapname);
+
+        GF_FREE (snap->description);
+        GF_FREE (snap);
+
+        return 0;
+}
+
+int32_t
+glusterd_snap_remove (dict_t *rsp_dict,
+                      glusterd_snap_t *snap,
+                      gf_boolean_t remove_lvm,
+                      gf_boolean_t force)
+{
+        int                 ret       = -1;
+        int                 save_ret  =  0;
+        glusterd_volinfo_t *snap_vol  = NULL;
+        glusterd_volinfo_t *tmp       = NULL;
+        xlator_t           *this      = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap);
+
+        if (!snap) {
+                gf_log(this->name, GF_LOG_WARNING, "snap is NULL");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
+                ret = glusterd_snap_volume_remove (rsp_dict, snap_vol,
+                                                   remove_lvm, force);
+                if (ret) {
+                        gf_log(this->name, GF_LOG_WARNING, "Failed to remove "
+                               "volinfo %s for snap %s", snap_vol->volname,
+                               snap->snapname);
+                        save_ret = ret;
+
+                        /* Continue to cleaning up the snap in case of error
+                           if force flag is enabled */
+                        if (!force)
+                                goto out;
+                }
+        }
+
+        ret = glusterd_store_delete_snap (snap);
+        if (ret) {
+                gf_log(this->name, GF_LOG_WARNING, "Failed to remove snap %s "
+                       "from store", snap->snapname);
+                save_ret = ret;
+                if (!force)
+                        goto out;
+        }
+
+        ret = glusterd_snapobject_delete (snap);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
+                        "snap object %s", snap->snapname);
+
+        if (save_ret)
+                ret = save_ret;
+out:
+        gf_log (THIS->name, GF_LOG_TRACE, "returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_snapshot_get_snapvol_detail (dict_t *dict,
+                                      glusterd_volinfo_t *snap_vol,
+                                      char *keyprefix, int detail)
+{
+        int                 ret           = -1;
+        int                 snap_limit    = 0;
+        char                key[PATH_MAX] = {0,};
+        char               *value         = NULL;
+        glusterd_volinfo_t *origin_vol    = NULL;
+        glusterd_conf_t    *conf          = NULL;
+        xlator_t           *this          = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (keyprefix);
+
+        /* Volume Name */
+        value = gf_strdup (snap_vol->volname);
+        if (!value)
+                goto out;
+
+        snprintf (key, sizeof (key), "%s.volname", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "volume name in dictionary: %s", key);
+                goto out;
+        }
+
+        /* Volume ID */
+        value = gf_strdup (uuid_utoa (snap_vol->volume_id));
+        if (NULL == value) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.vol-id", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "volume id in dictionary: %s", key);
+                goto out;
+        }
+        value = NULL;
+
+        /* volume status */
+        snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
+        switch (snap_vol->status) {
+        case GLUSTERD_STATUS_STARTED:
+                ret = dict_set_str (dict, key, "Started");
+                break;
+        case GLUSTERD_STATUS_STOPPED:
+                ret = dict_set_str (dict, key, "Stopped");
+                break;
+        case GD_SNAP_STATUS_NONE:
+                ret = dict_set_str (dict, key, "None");
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Invalid volume status");
+                ret = -1;
+                goto out;
+        }
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volume status"
+                        " in dictionary: %s", key);
+                goto out;
+        }
+
+
+        ret = glusterd_volinfo_find (snap_vol->parent_volname, &origin_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to get the parent "
+                        "volinfo for the volume %s", snap_vol->volname);
+                goto out;
+        }
+
+        /* Snaps available */
+        if (conf->snap_max_hard_limit < origin_vol->snap_max_hard_limit) {
+                snap_limit = conf->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "system snap-max-hard-limit is"
+                       " lesser than volume snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        } else {
+                snap_limit = origin_vol->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "volume snap-max-hard-limit is"
+                       " lesser than system snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        }
+
+        snprintf (key, sizeof (key), "%s.snaps-available", keyprefix);
+        if (snap_limit > origin_vol->snap_count)
+                ret = dict_set_int32 (dict, key,
+                        snap_limit - origin_vol->snap_count);
+        else
+                ret = dict_set_int32 (dict, key, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set available snaps");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.snapcount", keyprefix);
+        ret = dict_set_int32 (dict, key, origin_vol->snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save snapcount");
+                goto out;
+        }
+
+        if (!detail)
+                goto out;
+
+        /* Parent volume name */
+        value = gf_strdup (snap_vol->parent_volname);
+        if (!value)
+                goto out;
+
+        snprintf (key, sizeof (key), "%s.origin-volname", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set parent "
+                        "volume name in dictionary: %s", key);
+                goto out;
+        }
+        value = NULL;
+
+        ret = 0;
+out:
+        if (value)
+                GF_FREE (value);
+
+        return ret;
+}
+
+static int
+glusterd_snapshot_get_snap_detail (dict_t *dict, glusterd_snap_t *snap,
+                                char *keyprefix, glusterd_volinfo_t *volinfo)
+{
+        int                 ret           = -1;
+        int                 volcount      = 0;
+        char                key[PATH_MAX] = {0,};
+        char               *value         = NULL;
+        char               *timestr       = NULL;
+        struct tm          *tmptr         = NULL;
+        glusterd_volinfo_t *snap_vol      = NULL;
+        glusterd_volinfo_t *tmp_vol       = NULL;
+        xlator_t           *this          = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (snap);
+        GF_ASSERT (keyprefix);
+
+        /* Snap Name */
+        value = gf_strdup (snap->snapname);
+        if (!value)
+                goto out;
+
+        snprintf (key, sizeof (key), "%s.snapname", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "snap name in dictionary");
+                goto out;
+        }
+
+        /* Snap ID */
+        value = gf_strdup (uuid_utoa (snap->snap_id));
+        if (NULL == value) {
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.snap-id", keyprefix);
+        ret = dict_set_dynstr (dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "snap id in dictionary");
+                goto out;
+        }
+        value = NULL;
+
+        tmptr = localtime (&(snap->time_stamp));
+        if (NULL == tmptr) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to convert "
+                                "time_t to *tm");
+                ret = -1;
+                goto out;
+        }
+
+        timestr = GF_CALLOC (1, PATH_MAX, gf_gld_mt_char);
+        if (NULL == timestr) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = strftime (timestr, PATH_MAX, "%Y-%m-%d %H:%M:%S", tmptr);
+        if (0 == ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to convert time_t "
+                        "to string");
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%s.snap-time", keyprefix);
+        ret = dict_set_dynstr (dict, key, timestr);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "snap time stamp in dictionary");
+                goto out;
+        }
+        timestr = NULL;
+
+        /* If snap description is provided then add that into dictionary */
+        if (NULL != snap->description) {
+                value = gf_strdup (snap->description);
+                if (NULL == value) {
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "%s.snap-desc", keyprefix);
+                ret = dict_set_dynstr (dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                        "snap description in dictionary");
+                        goto out;
+                }
+                value = NULL;
+        }
+
+        snprintf (key, sizeof (key), "%s.snap-status", keyprefix);
+        switch (snap->snap_status) {
+        case GD_SNAP_STATUS_INIT:
+                ret = dict_set_str (dict, key, "Init");
+                break;
+        case GD_SNAP_STATUS_IN_USE:
+                ret = dict_set_str (dict, key, "In-use");
+                break;
+        case GD_SNAP_STATUS_DECOMMISSION:
+                ret = dict_set_str (dict, key, "Decommisioned");
+                break;
+        case GD_SNAP_STATUS_RESTORED:
+                ret = dict_set_str (dict, key, "Restored");
+                break;
+        case GD_SNAP_STATUS_NONE:
+                ret = dict_set_str (dict, key, "None");
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Invalid snap status");
+                ret = -1;
+                goto out;
+        }
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap status "
+                        "in dictionary");
+                goto out;
+        }
+
+        if (volinfo) {
+                volcount = 1;
+                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
+                ret = glusterd_snapshot_get_snapvol_detail (dict,
+                                                        volinfo, key, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "get volume detail %s for snap %s",
+                                snap_vol->volname, snap->snapname);
+                        goto out;
+                }
+                goto done;
+        }
+
+        list_for_each_entry_safe (snap_vol, tmp_vol, &snap->volumes, vol_list) {
+                volcount++;
+                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
+                ret = glusterd_snapshot_get_snapvol_detail (dict,
+                                                        snap_vol, key, 1);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "get volume detail %s for snap %s",
+                                snap_vol->volname, snap->snapname);
+                        goto out;
+                }
+        }
+
+done:
+        snprintf (key, sizeof (key), "%s.vol-count", keyprefix);
+        ret = dict_set_int32 (dict, key, volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
+                        key);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (value)
+                GF_FREE (value);
+
+        if (timestr)
+                GF_FREE(timestr);
+
+        return ret;
+}
+
+static int
+glusterd_snapshot_get_all_snap_info (dict_t *dict)
+{
+        int                 ret           = -1;
+        int                 snapcount     = 0;
+        char                key[PATH_MAX] = {0,};
+        glusterd_snap_t    *snap          = NULL;
+        glusterd_snap_t    *tmp_snap      = NULL;
+        glusterd_conf_t    *priv          = NULL;
+        xlator_t           *this          = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* General parameter validation */
+        GF_ASSERT (dict);
+
+        list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots, snap_list) {
+                snapcount++;
+                snprintf (key, sizeof (key), "snap%d", snapcount);
+                ret = glusterd_snapshot_get_snap_detail (dict, snap, key, NULL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "snapdetail for snap %s", snap->snapname);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_get_info_by_volume (dict_t *dict, char *volname,
+                                      char *err_str, size_t len)
+{
+        int                  ret           = -1;
+        int                  snapcount     = 0;
+        int                  snap_limit    = 0;
+        char                *value         = NULL;
+        char                 key[PATH_MAX] = "";
+        glusterd_volinfo_t  *volinfo       = NULL;
+        glusterd_volinfo_t  *snap_vol      = NULL;
+        glusterd_volinfo_t  *tmp_vol       = NULL;
+        glusterd_conf_t     *conf          = NULL;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (volname);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (err_str, len, "Volume (%s) does not exist", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        /* Snaps available */
+        if (conf->snap_max_hard_limit < volinfo->snap_max_hard_limit) {
+                snap_limit = conf->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "system snap-max-hard-limit is"
+                       " lesser than volume snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        } else {
+                snap_limit = volinfo->snap_max_hard_limit;
+                gf_log(this->name, GF_LOG_DEBUG, "volume snap-max-hard-limit is"
+                       " lesser than system snap-max-hard-limit, "
+                       "snap-max-hard-limit value is set to %d", snap_limit);
+        }
+
+        if (snap_limit > volinfo->snap_count)
+                ret = dict_set_int32 (dict, "snaps-available",
+                        snap_limit - volinfo->snap_count);
+        else
+                ret = dict_set_int32 (dict, "snaps-available", 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set available snaps");
+                goto out;
+        }
+
+        /* Origin volume name */
+        value = gf_strdup (volinfo->volname);
+        if (!value)
+                goto out;
+
+        ret = dict_set_dynstr (dict, "origin-volname", value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set parent "
+                        "volume name in dictionary: %s", key);
+                goto out;
+        }
+        value = NULL;
+
+        list_for_each_entry_safe (snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                  snapvol_list) {
+                snapcount++;
+                snprintf (key, sizeof (key), "snap%d", snapcount);
+                ret = glusterd_snapshot_get_snap_detail (dict,
+                                                         snap_vol->snapshot,
+                                                         key, snap_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "snapdetail for snap %s",
+                                snap_vol->snapshot->snapname);
+                        goto out;
+                }
+        }
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (value)
+                GF_FREE (value);
+
+        return ret;
+}
+
+/* This function will be called from RPC handler routine.
+ * This function is responsible for getting the requested
+ * snapshot info into the dictionary.
+ *
+ * @param req   RPC request object. Required for sending a response back.
+ * @param op    glusterd operation. Required for sending a response back.
+ * @param dict  pointer to dictionary which will contain both
+ *              request and response key-pair values.
+ * @return -1 on error and 0 on success
+ */
+int
+glusterd_handle_snapshot_info (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int              ret             =       -1;
+        int8_t          snap_driven      =        1;
+        char            *volname         =      NULL;
+        char            *snapname        =      NULL;
+        glusterd_snap_t *snap            =      NULL;
+        xlator_t        *this            =      NULL;
+        int32_t         cmd              =      GF_SNAP_INFO_TYPE_ALL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, req, out);
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get type "
+                        "of snapshot info");
+                goto out;
+        }
+
+        switch (cmd) {
+                case GF_SNAP_INFO_TYPE_ALL:
+                {
+                        ret = glusterd_snapshot_get_all_snap_info (dict);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get info of all snaps");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_INFO_TYPE_SNAP:
+                {
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get snap name");
+                                goto out;
+                        }
+
+                        ret = dict_set_int32 (dict, "snap-count", 1);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set snapcount");
+                                goto out;
+                        }
+
+                        snap = glusterd_find_snap_by_name (snapname);
+                        if (!snap) {
+                                snprintf (err_str, len,
+                                         "Snap (%s) does not exist", snapname);
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "%s", err_str);
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = glusterd_snapshot_get_snap_detail (dict, snap,
+                                                                "snap1", NULL);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get snap detail of snap "
+                                        "%s", snap->snapname);
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_INFO_TYPE_VOL:
+                {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get volname");
+                                goto out;
+                        }
+                        ret = glusterd_snapshot_get_info_by_volume (dict,
+                                                        volname, err_str, len);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to get volume info of volume "
+                                        "%s", volname);
+                                goto out;
+                        }
+                        snap_driven = 0;
+                        break;
+                }
+        }
+
+        ret = dict_set_int8 (dict, "snap-driven", snap_driven);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap-driven");
+                goto out;
+        }
+
+        /* If everything is successful then send the response back to cli.
+         * In case of failure the caller of this function will take care
+           of the response */
+        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
+                                "response");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+/* This function sets all the snapshot names in the dictionary */
+int
+glusterd_snapshot_get_all_snapnames (dict_t *dict)
+{
+        int              ret             = -1;
+        int              snapcount       = 0;
+        char            *snapname        = NULL;
+        char             key[PATH_MAX]   = {0,};
+        glusterd_snap_t *snap            = NULL;
+        glusterd_snap_t *tmp_snap        = NULL;
+        glusterd_conf_t *priv            = NULL;
+        xlator_t        *this            = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (dict);
+
+        list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots, snap_list) {
+                snapcount++;
+                snapname = gf_strdup (snap->snapname);
+                if (!snapname) {
+                        gf_log (this->name, GF_LOG_ERROR, "strdup failed");
+                        ret = -1;
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "snapname%d", snapcount);
+                ret = dict_set_dynstr (dict, key, snapname);
+                if (ret) {
+                        GF_FREE (snapname);
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
+                                key);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+
+/* This function sets all the snapshot names
+   under a given volume in the dictionary */
+int
+glusterd_snapshot_get_vol_snapnames (dict_t *dict, glusterd_volinfo_t *volinfo)
+{
+        int                 ret             = -1;
+        int                 snapcount       = 0;
+        char               *snapname        = NULL;
+        char                key[PATH_MAX]   = {0,};
+        glusterd_volinfo_t *snap_vol        = NULL;
+        glusterd_volinfo_t *tmp_vol         = NULL;
+        xlator_t           *this            = NULL;
+
+        this = THIS;
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry_safe (snap_vol, tmp_vol,
+                                  &volinfo->snap_volumes, snapvol_list) {
+                snapcount++;
+                snapname = gf_strdup (snap_vol->snapshot->snapname);
+                if (!snapname) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "strdup failed");
+                        ret = -1;
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "snapname%d", snapcount);
+                ret = dict_set_dynstr (dict, key, snapname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "set %s", key);
+                        GF_FREE (snapname);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dict, "snap-count", snapcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+
+int
+glusterd_handle_snapshot_list (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int                 ret             = -1;
+        char               *volname         = NULL;
+        glusterd_volinfo_t *volinfo         = NULL;
+        xlator_t           *this            = NULL;
+
+        this = THIS;
+
+        GF_VALIDATE_OR_GOTO (this->name, req, out);
+        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+
+        /* Ignore error for getting volname as it is optional */
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (NULL == volname) {
+                ret = glusterd_snapshot_get_all_snapnames (dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get snapshot list");
+                        goto out;
+                }
+        } else {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, len,
+                                 "Volume (%s) does not exist", volname);
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s", err_str);
+                        goto out;
+                }
+
+                ret = glusterd_snapshot_get_vol_snapnames (dict, volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get snapshot list for volume %s",
+                                volname);
+                        goto out;
+                }
+        }
+
+        /* If everything is successful then send the response back to cli.
+        In case of failure the caller of this function will take of response.*/
+        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
+                                "response");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+/* This is a snapshot create handler function. This function will be
+ * executed in the originator node. This function is responsible for
+ * calling mgmt_v3 framework to do the actual snap creation on all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot restore request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_handle_snapshot_create (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int           ret                     = -1;
+        char         *volname                 = NULL;
+        char         *snapname                = NULL;
+        int64_t       volcount                = 0;
+        xlator_t     *this                    = NULL;
+        char          key[PATH_MAX]           = "";
+        char         *username                = NULL;
+        char         *password                = NULL;
+        uuid_t       *uuid_ptr                = NULL;
+        uuid_t        tmp_uuid                = {0};
+        int           i                       = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_int64 (dict, "volcount", &volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                        "get the volume count");
+                goto out;
+        }
+        if (volcount <= 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid volume count %ld "
+                        "supplied", volcount);
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to get the snapname");
+                goto out;
+        }
+
+        if (strlen(snapname) >= GLUSTERD_MAX_SNAP_NAME) {
+                snprintf (err_str, len, "snapname cannot exceed 255 "
+                          "characters");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+        uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!uuid_ptr) {
+                gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
+                ret = -1;
+                goto out;
+        }
+
+        uuid_generate (*uuid_ptr);
+        ret = dict_set_bin (dict, "snap-id", uuid_ptr, sizeof(uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to set snap-id");
+                GF_FREE (uuid_ptr);
+                goto out;
+        }
+        uuid_ptr = NULL;
+
+        ret = dict_set_int64 (dict, "snap-time", (int64_t)time(NULL));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to set snap-time");
+                goto out;
+        }
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%d", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get volume name");
+                        goto out;
+                }
+
+                /* generate internal username and password  for the snap*/
+                uuid_generate (tmp_uuid);
+                username = gf_strdup (uuid_utoa (tmp_uuid));
+                snprintf (key, sizeof(key), "volume%d_username", i);
+                ret = dict_set_dynstr (dict, key, username);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
+                                "username for volume %s", volname);
+                        GF_FREE (username);
+                        goto out;
+                }
+
+                uuid_generate (tmp_uuid);
+                password = gf_strdup (uuid_utoa (tmp_uuid));
+                snprintf (key, sizeof(key), "volume%d_password", i);
+                ret = dict_set_dynstr (dict, key, password);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
+                                "password for volume %s", volname);
+                        GF_FREE (password);
+                        goto out;
+                }
+
+                uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+                if (!uuid_ptr) {
+                        gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof(key) - 1, "vol%d_volid", i);
+                uuid_generate (*uuid_ptr);
+                ret = dict_set_bin (dict, key, uuid_ptr, sizeof(uuid_t));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to set snap_volid");
+                        GF_FREE (uuid_ptr);
+                        goto out;
+                }
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
+                                "phases");
+        }
+
+out:
+        return ret;
+}
+
+/* This is a snapshot status handler function. This function will be
+ * executed in a originator node. This function is responsible for
+ * calling mgmt v3 framework to get the actual snapshot status from
+ * all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot status request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * return :  0  in case of success.
+ *          -1  in case of failure.
+ *
+ */
+int
+glusterd_handle_snapshot_status (rpcsvc_request_t *req, glusterd_op_t op,
+                                 dict_t *dict, char *err_str, size_t len)
+{
+        int                     ret             =       -1;
+        char                    *volname        =       NULL;
+        char                    *snapname       =       NULL;
+        char                    *buf            =       NULL;
+        glusterd_conf_t         *conf           =       NULL;
+        xlator_t                *this           =       NULL;
+        int32_t                 cmd             =       -1;
+        int                     i               =       0;
+        dict_t                  *voldict        =       NULL;
+        char                    key[PATH_MAX]   =       "";
+        glusterd_volinfo_t     *volinfo         =       NULL;
+        glusterd_snap_t         *snap           =       NULL;
+        glusterd_volinfo_t      *snap_volinfo   =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not get status type");
+                goto out;
+        }
+        switch (cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL:
+                {
+                        /* IF we give "gluster snapshot status"
+                         * then lock is held on all snaps.
+                         * This is the place where necessary information
+                         * (snapname and snapcount)is populated in dictionary
+                         * for locking.
+                         */
+                        ++i;
+                        list_for_each_entry (snap, &conf->snapshots, snap_list)
+                        {
+                                snprintf (key, sizeof (key), "snapname%d", i);
+                                buf = gf_strdup (snap->snapname);
+                                if (!buf) {
+                                        ret = -1;
+                                        goto out;
+                                }
+                                ret = dict_set_dynstr (dict, key, buf);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Could not save snapname (%s) "
+                                                "in the dictionary",
+                                                snap->snapname);
+                                        GF_FREE (buf);
+                                        goto out;
+                                }
+
+                                buf = NULL;
+                                i++;
+                        }
+
+                        ret = dict_set_int32 (dict, "snapcount", i - 1);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Could not "
+                                        "save snapcount in the dictionary");
+                                goto out;
+                        }
+                        break;
+                }
+
+                case GF_SNAP_STATUS_TYPE_SNAP:
+                {
+                        /* IF we give "gluster snapshot status <snapname>"
+                         * then lock is held on single snap.
+                         * This is the place where necessary information
+                         * (snapname)is populated in dictionary
+                         * for locking.
+                         */
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to fetch snap name");
+                                goto out;
+                        }
+
+                        snap = glusterd_find_snap_by_name (snapname);
+                        if (!snap) {
+                                snprintf (err_str, len, "Snap (%s)"
+                                          "does not exist", snapname);
+                                gf_log(this->name, GF_LOG_ERROR,
+                                        "%s", err_str);
+                                ret = -1;
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_VOL:
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to fetch volname");
+                                goto out;
+                        }
+
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret) {
+                                snprintf (err_str, len, "Volume (%s) "
+                                          "does not exist", volname);
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "%s", err_str);
+                                goto out;
+                        }
+
+                        i = 1;
+                        list_for_each_entry (snap_volinfo,
+                                        &volinfo->snap_volumes, snapvol_list) {
+                                snprintf (key, sizeof (key), "snapname%d", i);
+
+                                buf = gf_strdup
+                                             (snap_volinfo->snapshot->snapname);
+                                if (!buf) {
+                                        ret = -1;
+                                        goto out;
+                                }
+
+                                ret = dict_set_dynstr (dict, key, buf);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Could not save snapname");
+                                        GF_FREE (buf);
+                                        goto out;
+                                }
+
+                                buf = NULL;
+                                i++;
+                        }
+
+                        ret = dict_set_int32 (dict, "snapcount", i-1);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save snapcount");
+                                goto out;
+                        }
+                        break;
+                default:
+                {
+                        gf_log (this->name, GF_LOG_ERROR, "Unknown type");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Volume lock is not necessary for snapshot status, hence
+         * turning it off
+         */
+        ret = dict_set_int8 (dict, "hold_vol_locks", 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Setting volume lock "
+                        "flag failed");
+                goto out;
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate "
+                        "snap phases");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (voldict) {
+                dict_unref (voldict);
+        }
+        return ret;
+}
+
+
+/* This is a snapshot restore handler function. This function will be
+ * executed in the originator node. This function is responsible for
+ * calling mgmt_v3 framework to do the actual restore on all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot restore request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_handle_snapshot_restore (rpcsvc_request_t *req, glusterd_op_t op,
+                               dict_t *dict, char *err_str, size_t len)
+{
+        int                     ret             = -1;
+        char                    *snapname       = NULL;
+        char                    *buf            = NULL;
+        glusterd_conf_t         *conf           = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        glusterd_volinfo_t      *snap_volinfo   = NULL;
+        int32_t                 i               =    0;
+        char                    key[PATH_MAX]   =    "";
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                        "get snapname");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                snprintf (err_str, len, "Snap (%s) does not exist", snapname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (snap_volinfo, &snap->volumes, vol_list) {
+                i++;
+                snprintf (key, sizeof (key), "volname%d", i);
+                buf = gf_strdup (snap_volinfo->parent_volname);
+                if (!buf) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_dynstr (dict, key, buf);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not set "
+                                "parent volume name %s in the dict",
+                                snap_volinfo->parent_volname);
+                        GF_FREE (buf);
+                        goto out;
+                }
+                buf = NULL;
+        }
+
+        ret = dict_set_int32 (dict, "volcount", i);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not save volume count");
+                goto out;
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
+                                "phases");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+glusterd_snap_t*
+glusterd_create_snap_object (dict_t *dict, dict_t *rsp_dict)
+{
+        char              *snapname    = NULL;
+        uuid_t            *snap_id     = NULL;
+        char              *description = NULL;
+        glusterd_snap_t   *snap        = NULL;
+        xlator_t          *this        = NULL;
+        glusterd_conf_t   *priv        = NULL;
+        int                ret         = -1;
+        int64_t            time_stamp  = 0;
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        /* Fetch snapname, description, id and time from dict */
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snapname");
+                goto out;
+        }
+
+        /* Ignore ret value for description*/
+        ret = dict_get_str (dict, "description", &description);
+
+        ret = dict_get_bin (dict, "snap-id", (void **)&snap_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snap_id");
+                goto out;
+        }
+
+        ret = dict_get_int64 (dict, "snap-time", &time_stamp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snap-time");
+                goto out;
+        }
+        if (time_stamp <= 0) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Invalid time-stamp: %ld",
+                        time_stamp);
+                goto out;
+        }
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                if (!strcmp (snap->snapname, snapname) ||
+                    !uuid_compare (snap->snap_id, *snap_id)) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "Found duplicate snap %s (%s)",
+                                snap->snapname, uuid_utoa (snap->snap_id));
+                        ret = -1;
+                        break;
+                }
+        }
+        if (ret) {
+                snap = NULL;
+                goto out;
+        }
+
+        snap = glusterd_new_snap_object ();
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not create "
+                        "the snap object for snap %s", snapname);
+                goto out;
+        }
+
+        strcpy (snap->snapname, snapname);
+        uuid_copy (snap->snap_id, *snap_id);
+        snap->time_stamp = (time_t)time_stamp;
+        /* Set the status as GD_SNAP_STATUS_INIT and once the backend snapshot
+           is taken and snap is really ready to use, set the status to
+           GD_SNAP_STATUS_IN_USE. This helps in identifying the incomplete
+           snapshots and cleaning them up.
+        */
+        snap->snap_status = GD_SNAP_STATUS_INIT;
+        if (description) {
+                snap->description = gf_strdup (description);
+                if (snap->description == NULL) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Saving the Snap Description Failed");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = glusterd_store_snap (snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Could not store snap"
+                        "object %s", snap->snapname);
+                goto out;
+        }
+
+        list_add_order (&snap->snap_list, &priv->snapshots,
+                        glusterd_compare_snap_time);
+
+        gf_log (this->name, GF_LOG_TRACE, "Snap %s added to the list",
+                snap->snapname);
+
+        ret = 0;
+
+out:
+        if (ret) {
+                if (snap)
+                        glusterd_snap_remove (rsp_dict, snap,
+                                              _gf_true, _gf_true);
+                snap = NULL;
+        }
+
+        return snap;
+}
+
+/* This function is called to get the device path of the snap lvm. Usually
+   if /dev/mapper/<group-name>-<lvm-name> is the device for the lvm,
+   then the snap device will be /dev/<group-name>/<snapname>.
+   This function takes care of building the path for the snap device.
+*/
+char *
+glusterd_build_snap_device_path (char *device, char *snapname)
+{
+        char        snap[PATH_MAX]      = "";
+        char        msg[1024]           = "";
+        char        volgroup[PATH_MAX]  = "";
+        char       *snap_device         = NULL;
+        xlator_t   *this                = NULL;
+        runner_t    runner              = {0,};
+        char       *ptr                 = NULL;
+        int         ret                 = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR, "device is NULL");
+                goto out;
+        }
+        if (!snapname) {
+                gf_log (this->name, GF_LOG_ERROR, "snapname is NULL");
+                goto out;
+        }
+
+        runinit (&runner);
+        runner_add_args (&runner, "/sbin/lvs", "--noheadings", "-o", "vg_name",
+                         device, NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        snprintf (msg, sizeof (msg), "Get volume group for device %s", device);
+        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume group "
+                        "for device %s", device);
+                runner_end (&runner);
+                goto out;
+        }
+        ptr = fgets(volgroup, sizeof(volgroup),
+                    runner_chio (&runner, STDOUT_FILENO));
+        if (!ptr || !strlen(volgroup)) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume group "
+                        "for snap %s", snapname);
+                runner_end (&runner);
+                ret = -1;
+                goto out;
+        }
+        runner_end (&runner);
+
+        snprintf (snap, sizeof(snap), "/dev/%s/%s", gf_trim(volgroup),
+                  snapname);
+        snap_device = gf_strdup (snap);
+        if (!snap_device) {
+                gf_log (this->name, GF_LOG_WARNING, "Cannot copy the "
+                        "snapshot device name for snapname: %s)", snapname);
+        }
+
+out:
+        return snap_device;
+}
+
+/* This function actually calls the command (or the API) for taking the
+   snapshot of the backend brick filesystem. If this is successful,
+   then call the glusterd_snap_create function to create the snap object
+   for glusterd
+*/
+char *
+glusterd_take_lvm_snapshot (glusterd_volinfo_t *snap_vol,
+                            glusterd_brickinfo_t *brickinfo)
+{
+        char             msg[NAME_MAX]    = "";
+        char             buf[PATH_MAX]    = "";
+        char            *snap_device      = NULL;
+        char            *ptr              = NULL;
+        char            *device           = NULL;
+        int              ret              = -1;
+        gf_boolean_t     match            = _gf_false;
+        runner_t         runner           = {0,};
+        xlator_t        *this             = NULL;
+
+        this = THIS;
+
+        if (!brickinfo) {
+                gf_log (this->name, GF_LOG_ERROR, "brickinfo NULL");
+                goto out;
+        }
+
+        device = glusterd_get_brick_mount_details (brickinfo);
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR, "getting device name for "
+                        "the brick %s:%s failed", brickinfo->hostname,
+                        brickinfo->path);
+                goto out;
+        }
+
+        /* Figuring out if setactivationskip flag is supported or not */
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "running lvcreate help");
+        runner_add_args (&runner, "/sbin/lvcreate", "--help", NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to run lvcreate help");
+                runner_end (&runner);
+                goto out;
+        }
+
+        /* Looking for setactivationskip in lvcreate --help */
+        do {
+                ptr = fgets(buf, sizeof(buf),
+                            runner_chio (&runner, STDOUT_FILENO));
+                if (ptr) {
+                        if (strstr(buf, "setactivationskip")) {
+                                match = _gf_true;
+                                break;
+                        }
+                }
+        } while (ptr != NULL);
+        runner_end (&runner);
+
+        /* Takng the actual snapshot */
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "taking snapshot of the brick %s:%s",
+                  brickinfo->hostname, brickinfo->path);
+        if (match == _gf_true)
+                runner_add_args (&runner, "/sbin/lvcreate", "-s", device,
+                                 "--setactivationskip", "n", "--name",
+                                 snap_vol->volname, NULL);
+        else
+                runner_add_args (&runner, "/sbin/lvcreate", "-s", device,
+                                 "--name", snap_vol->volname, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "taking snapshot of the "
+                        "brick (%s:%s) of device %s failed",
+                        brickinfo->hostname, brickinfo->path, device);
+                runner_end (&runner);
+                goto out;
+        }
+        runner_end (&runner);
+
+        snap_device = glusterd_build_snap_device_path (device,
+                                                       snap_vol->volname);
+        if (!snap_device) {
+                gf_log (this->name, GF_LOG_WARNING, "Cannot copy the snapshot "
+                        "device name for snap %s (volume id: %s)",
+                        snap_vol->snapshot->snapname, snap_vol->volname);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        return snap_device;
+}
+
+int32_t
+glusterd_snap_brick_create (char *device, glusterd_volinfo_t *snap_volinfo,
+                            glusterd_brickinfo_t *original_brickinfo,
+                            int32_t brick_count, char *snap_brick_dir)
+{
+        int32_t          ret                             = -1;
+        xlator_t        *this                            = NULL;
+        glusterd_conf_t *priv                            = NULL;
+        char             snap_brick_mount_path[PATH_MAX] = "";
+        char             snap_brick_path[PATH_MAX]       = "";
+        char             msg[1024]                       = "";
+        struct stat      statbuf                         = {0, };
+        runner_t         runner                          = {0, };
+
+        this = THIS;
+        priv = this->private;
+
+        GF_ASSERT (device);
+        GF_ASSERT (snap_volinfo);
+        GF_ASSERT (original_brickinfo);
+        GF_ASSERT (snap_brick_dir);
+
+        snprintf (snap_brick_mount_path, sizeof (snap_brick_mount_path),
+                  "%s/%s/brick%d",  snap_mount_folder, snap_volinfo->volname,
+                  brick_count+1);
+
+        snprintf (snap_brick_path, sizeof (snap_brick_path), "%s%s",
+                  snap_brick_mount_path, snap_brick_dir);
+
+        ret = mkdir_p (snap_brick_mount_path, 0777, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "creating the brick directory"
+                        " %s for the snapshot %s(device: %s) failed",
+                        snap_brick_mount_path, snap_volinfo->volname, device);
+                goto out;
+        }
+        /* mount the snap logical device on the directory inside
+           /run/gluster/snaps/<snapname>/@snap_brick_mount_path
+           Way to mount the snap brick via mount api is this.
+           ret = mount (device, snap_brick_mount_path, entry->mnt_type,
+                        MS_MGC_VAL, "nouuid");
+           But for now, mounting using runner apis.
+        */
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "mounting snapshot of the brick %s:%s",
+                  original_brickinfo->hostname, original_brickinfo->path);
+        runner_add_args (&runner, "mount", "-o", "nouuid", device,
+                         snap_brick_mount_path, NULL);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+
+        /* let glusterd get blocked till snapshot is over */
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "mounting the snapshot "
+                        "logical device %s failed (error: %s)", device,
+                        strerror (errno));
+                goto out;
+        } else
+                gf_log (this->name, GF_LOG_DEBUG, "mounting the snapshot "
+                        "logical device %s successful", device);
+
+        ret = stat (snap_brick_path, &statbuf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "stat of the brick %s"
+                        "(brick mount: %s) failed (%s)", snap_brick_path,
+                        snap_brick_mount_path, strerror (errno));
+                goto out;
+        }
+        ret = sys_lsetxattr (snap_brick_path,
+                             GF_XATTR_VOL_ID_KEY,
+                             snap_volinfo->volume_id, 16,
+                             XATTR_REPLACE);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                        "extended attribute %s on %s. Reason: "
+                        "%s, snap: %s", GF_XATTR_VOL_ID_KEY,
+                        snap_brick_path, strerror (errno),
+                        snap_volinfo->volname);
+                goto out;
+        }
+
+out:
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "unmounting the snap brick"
+                        " mount %s", snap_brick_mount_path);
+                umount (snap_brick_mount_path);
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Added missed_snap_entry to rsp_dict */
+int32_t
+glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict, char *snap_uuid,
+                                   glusterd_brickinfo_t *brickinfo,
+                                   int32_t brick_number, int32_t op)
+{
+        char                   *buf                             = NULL;
+        char                    missed_snap_entry[PATH_MAX]     = "";
+        char                    name_buf[PATH_MAX]              = "";
+        int32_t                 missed_snap_count               = -1;
+        int32_t                 ret                             = -1;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_uuid);
+        GF_ASSERT (brickinfo);
+
+        snprintf (missed_snap_entry, sizeof(missed_snap_entry),
+                  "%s:%s=%d:%s:%d:%d", uuid_utoa(brickinfo->uuid),
+                  snap_uuid, brick_number, brickinfo->path, op,
+                  GD_MISSED_SNAP_PENDING);
+
+        buf = gf_strdup (missed_snap_entry);
+        if (!buf) {
+                ret = -1;
+                goto out;
+        }
+
+        /* Fetch the missed_snap_count from the dict */
+        ret = dict_get_int32 (rsp_dict, "missed_snap_count",
+                              &missed_snap_count);
+        if (ret) {
+                /* Initialize the missed_snap_count for the first time */
+                missed_snap_count = 0;
+        }
+
+        /* Setting the missed_snap_entry in the rsp_dict */
+        snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                  missed_snap_count);
+        ret = dict_set_dynstr (rsp_dict, name_buf, buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set missed_snap_entry (%s) "
+                        "in the rsp_dict.", buf);
+                GF_FREE (buf);
+                goto out;
+        }
+        missed_snap_count++;
+
+        /* Setting the new missed_snap_count in the dict */
+        ret = dict_set_int32 (rsp_dict, "missed_snap_count",
+                              missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set missed_snap_count for %s "
+                        "in the rsp_dict.", missed_snap_entry);
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_add_bricks_to_snap_volume (dict_t *dict, dict_t *rsp_dict,
+                                    glusterd_volinfo_t  *snap_vol,
+                                    glusterd_brickinfo_t *original_brickinfo,
+                                    glusterd_brickinfo_t *snap_brickinfo,
+                                    char **snap_brick_dir, int64_t volcount,
+                                    int32_t brick_count)
+{
+        char                    key[PATH_MAX]                   = "";
+        char                    snap_brick_path[PATH_MAX]       = "";
+        char                   *snap_device                     = NULL;
+        gf_boolean_t            add_missed_snap                 = _gf_false;
+        int32_t                 ret                             = -1;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (original_brickinfo);
+        GF_ASSERT (snap_brickinfo);
+        GF_ASSERT (snap_brick_dir);
+
+        snprintf (key, sizeof(key) - 1, "vol%ld.brickdir%d", volcount,
+                  brick_count);
+        ret = dict_get_ptr (dict, key, (void **)snap_brick_dir);
+        if (ret) {
+                /* Using original brickinfo here because it will be a
+                 * pending snapshot and storing the original brickinfo
+                 * will help in mapping while recreating the missed snapshot
+                 */
+                gf_log (this->name, GF_LOG_WARNING, "Unable to fetch "
+                        "snap mount path (%s). Using original brickinfo", key);
+                snap_brickinfo->snap_status = -1;
+                strcpy (snap_brick_path, original_brickinfo->path);
+
+                /* In origiator node add snaps missed
+                 * from different nodes to the dict
+                 */
+                if (is_origin_glusterd (dict) == _gf_true)
+                        add_missed_snap = _gf_true;
+        } else {
+                /* Create brick-path in the format /var/run/gluster/snaps/ *
+                 * <snap-uuid>/<original-brick#>/snap-brick-dir *
+                 */
+                snprintf (snap_brick_path, sizeof(snap_brick_path),
+                          "%s/%s/brick%d%s", snap_mount_folder,
+                          snap_vol->volname, brick_count+1,
+                          *snap_brick_dir);
+        }
+
+        if ((snap_brickinfo->snap_status != -1) &&
+            (!uuid_compare (original_brickinfo->uuid, MY_UUID)) &&
+            (!glusterd_is_brick_started (original_brickinfo))) {
+                /* In case if the brick goes down after prevalidate. */
+                gf_log (this->name, GF_LOG_WARNING, "brick %s:%s is not"
+                        " started (snap: %s)",
+                        original_brickinfo->hostname,
+                        original_brickinfo->path,
+                        snap_vol->snapshot->snapname);
+
+                snap_brickinfo->snap_status = -1;
+                strcpy (snap_brick_path, original_brickinfo->path);
+                add_missed_snap = _gf_true;
+        }
+
+        if (add_missed_snap) {
+                ret = glusterd_add_missed_snaps_to_dict (rsp_dict,
+                                            snap_vol->volname,
+                                            original_brickinfo,
+                                            brick_count + 1,
+                                            GF_SNAP_OPTION_TYPE_CREATE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to add missed"
+                                " snapshot info for %s:%s in the rsp_dict",
+                                original_brickinfo->hostname,
+                                original_brickinfo->path);
+                        goto out;
+                }
+        }
+
+        snprintf (key, sizeof(key), "vol%ld.brick_snapdevice%d",
+                  volcount, brick_count);
+        ret = dict_get_ptr (dict, key, (void **)&snap_device);
+        if (ret) {
+                /* If the device name is empty, so will be the brick path
+                 * Hence the missed snap has already been added above
+                 */
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
+                        "snap device (%s). Leaving empty", key);
+        } else
+                strcpy (snap_brickinfo->device_path, snap_device);
+
+        ret = gf_canonicalize_path (snap_brick_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to canonicalize path");
+                goto out;
+        }
+
+        strcpy (snap_brickinfo->hostname, original_brickinfo->hostname);
+        strcpy (snap_brickinfo->path, snap_brick_path);
+        uuid_copy (snap_brickinfo->uuid, original_brickinfo->uuid);
+        list_add_tail (&snap_brickinfo->brick_list, &snap_vol->bricks);
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_take_brick_snapshot (glusterd_volinfo_t *origin_vol,
+                              glusterd_volinfo_t *snap_vol, dict_t *rsp_dict,
+                              glusterd_brickinfo_t *original_brickinfo,
+                              glusterd_brickinfo_t *snap_brickinfo,
+                              char *snap_brick_dir, int32_t brick_count)
+{
+        char                   *device                          = NULL;
+        int32_t                 ret                             = -1;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (origin_vol);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (original_brickinfo);
+        GF_ASSERT (snap_brickinfo);
+        GF_ASSERT (snap_brick_dir);
+
+        device = glusterd_take_lvm_snapshot (snap_vol, original_brickinfo);
+        /* Fail the snapshot even though snapshot on one of
+           the bricks fails. At the end when we check whether
+           the snapshot volume meets quorum or not, then the
+           the snapshot can either be treated as success, or
+           in case of failure we can undo the changes and return
+           failure to cli.  */
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to take snapshot of %s:%s",
+                        original_brickinfo->hostname,
+                        original_brickinfo->path);
+                goto out;
+        }
+
+        /* create the complete brick here */
+        ret = glusterd_snap_brick_create (device, snap_vol,
+                                          original_brickinfo,
+                                          brick_count, snap_brick_dir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "not able to"
+                        " create the brickinfo for the snap %s"
+                        ", volume %s", snap_vol->snapshot->snapname,
+                        origin_vol->volname);
+                goto out;
+        }
+
+out:
+        if (device)
+                GF_FREE (device);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Look for disconnected peers, for missed snap creates or deletes */
+static int32_t
+glusterd_find_missed_snap (dict_t *rsp_dict, glusterd_volinfo_t *vol,
+                           char *snap_uuid, struct list_head *peers,
+                           int32_t op)
+{
+        int32_t                   brick_count          = -1;
+        int32_t                   ret                  = -1;
+        xlator_t                 *this                 = NULL;
+        glusterd_peerinfo_t      *peerinfo             = NULL;
+        glusterd_brickinfo_t     *brickinfo            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (peers);
+        GF_ASSERT (vol);
+        GF_ASSERT (snap_uuid);
+
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &vol->bricks, brick_list) {
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        /* If the brick belongs to the same node */
+                        brick_count++;
+                        continue;
+                }
+
+                list_for_each_entry (peerinfo, peers, uuid_list) {
+                        if (uuid_compare (peerinfo->uuid, brickinfo->uuid)) {
+                                /* If the brick doesnt belong to this peer */
+                                continue;
+                        }
+
+                        /* Found peer who owns the brick,    *
+                         * if peer is not connected or not   *
+                         * friend add it to missed snap list */
+                        if (!(peerinfo->connected) ||
+                           (peerinfo->state.state !=
+                             GD_FRIEND_STATE_BEFRIENDED)) {
+                                ret = glusterd_add_missed_snaps_to_dict
+                                                   (rsp_dict,
+                                                    snap_uuid,
+                                                    brickinfo,
+                                                    brick_count + 1,
+                                                    op);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to add missed snapshot "
+                                                "info for %s:%s in the "
+                                                "rsp_dict", brickinfo->hostname,
+                                                brickinfo->path);
+                                        goto out;
+                                }
+                        }
+                }
+                brick_count++;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+glusterd_volinfo_t *
+glusterd_do_snap_vol (glusterd_volinfo_t *origin_vol, glusterd_snap_t *snap,
+                      dict_t *dict, dict_t *rsp_dict, int64_t volcount)
+{
+        char                    key[PATH_MAX]                   = "";
+        char                   *snap_brick_dir                  = NULL;
+        char                   *username                        = NULL;
+        char                   *password                        = NULL;
+        glusterd_brickinfo_t   *brickinfo                       = NULL;
+        glusterd_conf_t        *priv                            = NULL;
+        glusterd_volinfo_t     *snap_vol                        = NULL;
+        uuid_t                 *snap_volid                      = NULL;
+        int32_t                 ret                             = -1;
+        int32_t                 brick_count                     = 0;
+        glusterd_brickinfo_t   *snap_brickinfo                  = NULL;
+        xlator_t               *this                            = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (origin_vol);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        /* fetch username, password and vol_id from dict*/
+        snprintf (key, sizeof(key), "volume%ld_username", volcount);
+        ret = dict_get_str (dict, key, &username);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get %s for "
+                        "snap %s", key, snap->snapname);
+                goto out;
+        }
+
+        snprintf (key, sizeof(key), "volume%ld_password", volcount);
+        ret = dict_get_str (dict, key, &password);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get %s for "
+                        "snap %s", key, snap->snapname);
+                goto out;
+        }
+        snprintf (key, sizeof(key) - 1, "vol%ld_volid", volcount);
+        ret = dict_get_bin (dict, key, (void **)&snap_volid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to fetch snap_volid");
+                goto out;
+        }
+
+        /* We are not setting the username and password here as
+         * we need to set the user name and password passed in
+         * the dictionary
+         */
+        ret = glusterd_volinfo_dup (origin_vol, &snap_vol, _gf_false);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to duplicate volinfo "
+                        "for the snapshot %s", snap->snapname);
+                goto out;
+        }
+
+        /* uuid is used as lvm snapshot name.
+           This will avoid restrictions on snapshot names provided by user */
+        GLUSTERD_GET_UUID_NOHYPHEN (snap_vol->volname, *snap_volid);
+        uuid_copy (snap_vol->volume_id, *snap_volid);
+        snap_vol->is_snap_volume = _gf_true;
+        strcpy (snap_vol->parent_volname, origin_vol->volname);
+        snap_vol->snapshot = snap;
+
+        glusterd_auth_set_username (snap_vol, username);
+        glusterd_auth_set_password (snap_vol, password);
+
+        /* Adding snap brickinfos to the snap volinfo */
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &origin_vol->bricks, brick_list) {
+                snap_brickinfo = NULL;
+
+                ret = glusterd_brickinfo_new (&snap_brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "initializing the brick for the snap "
+                                "volume failed (snapname: %s)", snap->snapname);
+                        goto out;
+                }
+
+                ret = glusterd_add_bricks_to_snap_volume (dict, rsp_dict,
+                                                          snap_vol,
+                                                          brickinfo,
+                                                          snap_brickinfo,
+                                                          &snap_brick_dir,
+                                                          volcount,
+                                                          brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to add the snap brick for "
+                                "%s:%s to the snap volume",
+                                brickinfo->hostname, brickinfo->path);
+                        GF_FREE (snap_brickinfo);
+                        goto out;
+                }
+
+                /* Take snapshot of the brick */
+                if ((uuid_compare (brickinfo->uuid, MY_UUID)) ||
+                    (snap_brickinfo->snap_status == -1)) {
+                        brick_count++;
+                        continue;
+                }
+
+                ret = glusterd_take_brick_snapshot (origin_vol, snap_vol,
+                                                    rsp_dict, brickinfo,
+                                                    snap_brickinfo,
+                                                    snap_brick_dir,
+                                                    brick_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to take snapshot for %s:%s",
+                                brickinfo->hostname, brickinfo->path);
+                        goto out;
+                }
+
+                brick_count++;
+        }
+
+        /*TODO: the quorum check of the snap volume here */
+
+        ret = glusterd_store_volinfo (snap_vol,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store snapshot "
+                        "volinfo (%s) for snap %s", snap_vol->volname,
+                        snap->snapname);
+                goto out;
+        }
+
+        ret = generate_brick_volfiles (snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "generating the brick "
+                        "volfiles for the snap %s (volume: %s) failed",
+                        snap->snapname, origin_vol->volname);
+                goto out;
+        }
+
+        ret = generate_client_volfiles (snap_vol, GF_CLIENT_TRUSTED);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "generating the trusted "
+                        "client volfiles for the snap %s (volume: %s) failed",
+                        snap->snapname, origin_vol->volname);
+                goto out;
+        }
+        ret = generate_client_volfiles (snap_vol, GF_CLIENT_OTHER);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "generating the client "
+                        "volfiles for the snap %s (volume: %s) failed",
+                        snap->snapname, origin_vol->volname);
+                goto out;
+        }
+
+        ret = glusterd_list_add_snapvol (origin_vol, snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "could not add the snap "
+                        "volume %s to the list", snap_vol->volname);
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                if (brickinfo->snap_status == -1) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "not starting snap brick %s:%s for "
+                                "for the snap %s (volume: %s)",
+                                brickinfo->hostname, brickinfo->path,
+                                snap->snapname, origin_vol->volname);
+                        continue;
+                }
+
+                ret = glusterd_brick_start (snap_vol, brickinfo, _gf_true);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "starting the "
+                                "brick %s:%s for the snap %s (volume: %s) "
+                                "failed", brickinfo->hostname, brickinfo->path,
+                                snap->snapname, origin_vol->volname);
+                        goto out;
+                }
+        }
+
+        snap_vol->status = GLUSTERD_STATUS_STARTED;
+        ret = glusterd_store_volinfo (snap_vol,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to store snap volinfo");
+                goto out;
+        }
+
+out:
+        if (ret) {
+                if (snap_vol)
+                        glusterd_snap_volume_remove (rsp_dict, snap_vol,
+                                                     _gf_true, _gf_true);
+                snap_vol = NULL;
+        }
+
+        return snap_vol;
+}
+
+/* This is a snapshot remove handler function. This function will be
+ * executed in the originator node. This function is responsible for
+ * calling mgmt v3 framework to do the actual remove on all the bricks
+ *
+ * @param req           RPC request object
+ * @param op            gluster operation
+ * @param dict          dictionary containing snapshot remove request
+ * @param err_str       In case of an err this string should be populated
+ * @param len           length of err_str buffer
+ *
+ * @return              Negative value on Failure and 0 in success
+ */
+int
+glusterd_handle_snapshot_remove (rpcsvc_request_t *req, glusterd_op_t op,
+                                 dict_t *dict, char *err_str, size_t len)
+{
+        int                      ret            = -1;
+        int64_t                  volcount       = 0;
+        char                    *snapname       = NULL;
+        char                    *volname        = NULL;
+        char                     key[PATH_MAX]  = "";
+        glusterd_snap_t         *snap           = NULL;
+        glusterd_volinfo_t      *snap_vol       = NULL;
+        glusterd_volinfo_t      *tmp            = NULL;
+        xlator_t                *this           = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (req);
+        GF_ASSERT (dict);
+        GF_ASSERT (err_str);
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get snapname");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                snprintf (err_str, len, "Snap (%s) does not exist", snapname);
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s", err_str);
+                ret = -1;
+                goto out;
+        }
+
+        /* Set volnames in the dict to get mgmt_v3 lock */
+        list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
+                volcount++;
+                volname = gf_strdup (snap_vol->parent_volname);
+                if (!volname) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR, "strdup failed");
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "volname%ld", volcount);
+                ret = dict_set_dynstr (dict, key, volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "volume name in dictionary");
+                        GF_FREE (volname);
+                        goto out;
+                }
+                volname = NULL;
+        }
+        ret = dict_set_int64 (dict, "volcount", volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
+                goto out;
+        }
+
+        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
+                        "phases");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_remove_prevalidate (dict_t *dict, char **op_errstr,
+                                      dict_t *rsp_dict)
+{
+        int32_t             ret          = -1;
+        char               *snapname     = NULL;
+        xlator_t           *this        = NULL;
+        glusterd_snap_t    *snap        = NULL;
+
+        this = THIS;
+
+        if (!dict || !op_errstr) {
+                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
+                        "failed");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Snap %s does not exist",
+                        snapname);
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_status_prevalidate (dict_t *dict, char **op_errstr,
+                                      dict_t *rsp_dict)
+{
+        int                     ret             =       -1;
+        char                    *snapname       =       NULL;
+        glusterd_conf_t         *conf           =       NULL;
+        xlator_t                *this           =       NULL;
+        int32_t                 cmd             =       -1;
+        glusterd_volinfo_t      *volinfo        =       NULL;
+        char                    *volname        =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        GF_ASSERT (op_errstr);
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Input dict is NULL");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not fetch status cmd");
+                goto out;
+        }
+
+        switch (cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL:
+                {
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_SNAP:
+                {
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not fetch snapname");
+                                goto out;
+                        }
+
+                        if (!glusterd_find_snap_by_name (snapname)) {
+                                ret = gf_asprintf (op_errstr, "Snap (%s) "
+                                                  "not found", snapname);
+                                if (ret < 0) {
+                                        goto out;
+                                }
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "Snap (%s) "
+                                        "not found", snapname);
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_VOL:
+                {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not fetch volname");
+                                goto out;
+                        }
+
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret) {
+                                ret = gf_asprintf (op_errstr, "Volume (%s)"
+                                                  "not found", volname);
+                                if (ret < 0) {
+                                        goto out;
+                                }
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "Volume "
+                                        "%s not present", volname);
+                                goto out;
+                        }
+                        break;
+
+                }
+                default:
+                {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid command");
+                        break;
+                }
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snapshot_remove_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        int32_t                   ret                  = -1;
+        char                     *snapname             = NULL;
+        char                     *dup_snapname         = NULL;
+        glusterd_snap_t          *snap                 = NULL;
+        glusterd_conf_t          *priv                 = NULL;
+        glusterd_volinfo_t       *snap_volinfo         = NULL;
+        xlator_t                 *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_errstr);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!dict || !op_errstr) {
+                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
+                        "failed");
+                goto out;
+        }
+
+        snap = glusterd_find_snap_by_name (snapname);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Snap %s does not exist",
+                        snapname);
+                ret = -1;
+                goto out;
+        }
+
+        if (is_origin_glusterd (dict) == _gf_true) {
+                /* TODO : As of now there is only volume in snapshot.
+                 * Change this when multiple volume snapshot is introduced
+                 */
+                snap_volinfo = list_entry (snap->volumes.next,
+                                           glusterd_volinfo_t,
+                                           vol_list);
+                if (!snap_volinfo) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch snap_volinfo");
+                        ret = -1;
+                        goto out;
+                }
+
+                /* From origin glusterd check if      *
+                 * any peers with snap bricks is down */
+                ret = glusterd_find_missed_snap (rsp_dict, snap_volinfo,
+                                                 snap_volinfo->volname,
+                                                 &priv->peers,
+                                                 GF_SNAP_OPTION_TYPE_DELETE);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to find missed snap deletes");
+                        goto out;
+                }
+        }
+
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_false);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to remove snap %s",
+                        snapname);
+                goto out;
+        }
+
+        dup_snapname = gf_strdup (snapname);
+        if (!dup_snapname) {
+                gf_log (this->name, GF_LOG_ERROR, "Strdup failed");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, "snapname", dup_snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set the snapname");
+                GF_FREE (dup_snapname);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_do_snap_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t                  ret                   = -1;
+        char                     *name                 = NULL;
+        xlator_t                 *this                 = NULL;
+        glusterd_conf_t          *conf                 = NULL;
+        glusterd_volinfo_t       *volinfo              = NULL;
+        glusterd_snap_t          *snap                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (!dict || !op_errstr) {
+                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "getting the snap "
+                        "name failed (volume: %s)", volinfo->volname);
+                goto out;
+        }
+
+        /*
+          If the snapname is not found that means the failure happened at
+          staging, or in commit, before the snap object is created, in which
+          case there is nothing to cleanup. So set ret to 0.
+        */
+        snap = glusterd_find_snap_by_name (name);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_INFO, "snap %s is not found", name);
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "removing the snap %s failed",
+                        name);
+                goto out;
+        }
+
+        name = NULL;
+
+        ret = 0;
+
+out:
+
+        return ret;
+}
+
+/* In case of a successful, delete or create operation, during post_validate *
+ * look for missed snap operations and update the missed snap lists */
+int32_t
+glusterd_snapshot_update_snaps_post_validate (dict_t *dict, char **op_errstr,
+                                              dict_t *rsp_dict)
+{
+        int32_t                  ret                    = -1;
+        int32_t                  missed_snap_count      = -1;
+        xlator_t                *this                   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_int32 (dict, "missed_snap_count",
+                              &missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_store_update_missed_snaps (dict, missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to update missed_snaps_list");
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        int                     ret                     = -1;
+        int64_t                 i                       = 0;
+        int64_t                 volcount                = 0;
+        char                    *snapname               = NULL;
+        char                    *volname                = NULL;
+        char                    *tmp_name               = NULL;
+        char                    key[PATH_MAX]           = "";
+        xlator_t                *this                   = NULL;
+        glusterd_snap_t         *snap                   = NULL;
+        glusterd_volinfo_t      *origin_vol             = NULL;
+        glusterd_volinfo_t      *snap_vol               = NULL;
+        glusterd_conf_t *priv   = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+        GF_ASSERT(op_errstr);
+        GF_ASSERT(rsp_dict);
+        priv = this->private;
+        GF_ASSERT(priv);
+
+        ret = dict_get_int64 (dict, "volcount", &volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                        "get the volume count");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "snapname", &snapname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snapname");
+                goto out;
+        }
+        tmp_name = gf_strdup (snapname);
+        if (!tmp_name) {
+                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, "snapname", tmp_name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set snapname in rsp_dict");
+                GF_FREE (tmp_name);
+                goto out;
+        }
+        tmp_name = NULL;
+
+        snap = glusterd_create_snap_object (dict, rsp_dict);
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "creating the"
+                        "snap object %s failed", snapname);
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%ld", i);
+                ret = dict_get_str (dict, key, &volname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to get volume name");
+                        goto out;
+                }
+
+                ret = glusterd_volinfo_find (volname, &origin_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to get the volinfo for "
+                                "the volume %s", volname);
+                        goto out;
+                }
+
+                /* TODO: Create a stub where the bricks are
+                   added parallely by worker threads so that
+                   the snap creating happens parallely.  */
+                snap_vol = glusterd_do_snap_vol (origin_vol, snap, dict,
+                                                 rsp_dict, i);
+                if (!snap_vol) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_WARNING, "taking the "
+                                "snapshot of the volume %s failed", volname);
+                        goto out;
+                }
+        }
+
+        snap->snap_status = GD_SNAP_STATUS_IN_USE;
+        ret = glusterd_store_snap (snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Could not store snap"
+                        "object %s", snap->snapname);
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (ret) {
+                if (snap)
+                        glusterd_snap_remove (rsp_dict, snap,
+                                              _gf_true, _gf_true);
+                snap = NULL;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+snap_max_hard_limit_set_commit (dict_t *dict, uint64_t value,
+                                char *volname, char **op_errstr)
+{
+        char                err_str[PATH_MAX]    = "";
+        glusterd_conf_t    *conf                 = NULL;
+        glusterd_volinfo_t *volinfo              = NULL;
+        int                 ret                  = -1;
+        xlator_t           *this                 = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (volname);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        /* TODO: Initiate auto deletion when there is a limit change */
+        if (!volname) {
+                /* For system limit */
+                conf->snap_max_hard_limit = value;
+
+                ret = glusterd_store_global_info (this);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to store "
+                                  "snap-max-hard-limit for system");
+                        goto out;
+                }
+        } else {
+               /*  For one volume */
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to get the"
+                                   " volinfo for volume %s", volname);
+                        goto out;
+                }
+
+                volinfo->snap_max_hard_limit = value;
+
+                ret = glusterd_store_volinfo (volinfo,
+                                        GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to store "
+                                 "snap-max-hard-limit for volume %s", volname);
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                *op_errstr = gf_strdup (err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+        }
+        return ret;
+}
+
+int
+snap_max_limits_display_commit (dict_t *rsp_dict, char *volname,
+                                char **op_errstr)
+{
+        char                err_str[PATH_MAX]    = "";
+        char                buf[PATH_MAX]        = "";
+        glusterd_conf_t    *conf                 = NULL;
+        glusterd_volinfo_t *volinfo              = NULL;
+        int                 ret                  = -1;
+        uint64_t            active_hard_limit    = 0;
+        uint64_t            snap_max_limit       = 0;
+        uint64_t            soft_limit_value     = -1;
+        uint64_t            count                = 0;
+        xlator_t           *this                 = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (volname);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        if (!volname) {
+                /* For system limit */
+                list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                        if (volinfo->is_snap_volume == _gf_true)
+                                continue;
+                        snap_max_limit = volinfo->snap_max_hard_limit;
+                        if (snap_max_limit > conf->snap_max_hard_limit)
+                                active_hard_limit = conf->snap_max_hard_limit;
+                        else
+                                active_hard_limit = snap_max_limit;
+                        soft_limit_value = (active_hard_limit *
+                                            conf->snap_max_soft_limit) / 100;
+
+                        snprintf (buf, sizeof(buf), "volume%ld-volname", count);
+                        ret = dict_set_str (rsp_dict, buf, volinfo->volname);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-hard-limit", count);
+                        ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-active-hard-limit", count);
+                        ret = dict_set_uint64 (rsp_dict, buf,
+                                               active_hard_limit);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-soft-limit", count);
+                        ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX,
+                                          "Failed to set %s", buf);
+                                goto out;
+                        }
+                        count++;
+                }
+
+                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set voldisplaycount");
+                        goto out;
+                }
+        } else {
+                /*  For one volume */
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX, "Failed to get the"
+                                   " volinfo for volume %s", volname);
+                        goto out;
+                }
+
+                snap_max_limit = volinfo->snap_max_hard_limit;
+                if (snap_max_limit > conf->snap_max_hard_limit)
+                        active_hard_limit = conf->snap_max_hard_limit;
+                else
+                        active_hard_limit = snap_max_limit;
+
+                soft_limit_value = (active_hard_limit *
+                                    conf->snap_max_soft_limit) / 100;
+
+                snprintf (buf, sizeof(buf), "volume%ld-volname", count);
+                ret = dict_set_str (rsp_dict, buf, volinfo->volname);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                snprintf (buf, sizeof(buf),
+                          "volume%ld-snap-max-hard-limit", count);
+                ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                snprintf (buf, sizeof(buf),
+                          "volume%ld-active-hard-limit", count);
+                ret = dict_set_uint64 (rsp_dict, buf, active_hard_limit);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                snprintf (buf, sizeof(buf),
+                          "volume%ld-snap-max-soft-limit", count);
+                ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set %s", buf);
+                        goto out;
+                }
+
+                count++;
+
+                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
+                if (ret) {
+                        snprintf (err_str, PATH_MAX,
+                                  "Failed to set voldisplaycount");
+                        goto out;
+                }
+
+        }
+
+        ret = dict_set_uint64 (rsp_dict, "snap-max-hard-limit",
+                               conf->snap_max_hard_limit);
+        if (ret) {
+                snprintf (err_str, PATH_MAX,
+                          "Failed to set sys-snap-max-hard-limit ");
+                goto out;
+        }
+
+        ret = dict_set_uint64 (rsp_dict, "snap-max-soft-limit",
+                               conf->snap_max_soft_limit);
+        if (ret) {
+                snprintf (err_str, PATH_MAX,
+                          "Failed to set sys-snap-max-hard-limit ");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                *op_errstr = gf_strdup (err_str);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+        }
+        return ret;
+}
+
+int
+glusterd_snapshot_config_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        char               *volname              = NULL;
+        xlator_t           *this                 = NULL;
+        int                 ret                  = -1;
+        char                err_str[PATH_MAX]    = {0,};
+        glusterd_conf_t    *conf                 = NULL;
+        int                 config_command       = 0;
+        uint64_t            hard_limit           = 0;
+        uint64_t            soft_limit           = 0;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+
+        ret = dict_get_int32 (dict, "config-command", &config_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get config-command type");
+                goto out;
+        }
+
+        /* Ignore the return value of the following dict_get,
+         * as they are optional
+         */
+        ret = dict_get_str (dict, "volname", &volname);
+
+        ret = dict_get_uint64 (dict, "snap-max-hard-limit", &hard_limit);
+
+        ret = dict_get_uint64 (dict, "snap-max-soft-limit", &soft_limit);
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_TYPE_SET:
+                if (hard_limit) {
+                        /* Commit ops for snap-max-hard-limit */
+                        ret = snap_max_hard_limit_set_commit (dict, hard_limit,
+                                                              volname,
+                                                              op_errstr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "snap-max-hard-limit set "
+                                        "commit failed.");
+                                goto out;
+                        }
+                }
+
+                if (soft_limit) {
+                        /* For system limit */
+                        conf->snap_max_soft_limit = soft_limit;
+
+                        ret = glusterd_store_global_info (this);
+                        if (ret) {
+                                snprintf (err_str, PATH_MAX, "Failed to store "
+                                          "snap-max-soft-limit for system");
+                                *op_errstr = gf_strdup (err_str);
+                                gf_log (this->name, GF_LOG_ERROR, "%s",
+                                        err_str);
+                                goto out;
+                        }
+                }
+                break;
+
+        case GF_SNAP_CONFIG_DISPLAY:
+                /* Reading data from local node only */
+                if (!is_origin_glusterd (dict)) {
+                        ret = 0;
+                        break;
+                }
+
+                ret = snap_max_limits_display_commit (rsp_dict, volname,
+                                                      op_errstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "snap-max-limit "
+                                "display commit failed.");
+                        goto out;
+                }
+                break;
+        default:
+                break;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_get_brick_lvm_details (dict_t *rsp_dict,
+                               glusterd_brickinfo_t *brickinfo, char *volname,
+                                char *device, char *key_prefix)
+{
+
+        int                     ret             =       -1;
+        glusterd_conf_t         *priv           =       NULL;
+        runner_t                runner          =       {0,};
+        xlator_t                *this           =       NULL;
+        char                    msg[PATH_MAX]   =       "";
+        char                    buf[PATH_MAX]   =       "";
+        char                    *ptr            =       NULL;
+        char                    *token          =       NULL;
+        char                    key[PATH_MAX]   =       "";
+        char                    *value          =       NULL;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (volname);
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        device = glusterd_get_brick_mount_details (brickinfo);
+        if (!device) {
+                gf_log (this->name, GF_LOG_ERROR, "Getting device name for "
+                        "the brick %s:%s failed", brickinfo->hostname,
+                         brickinfo->path);
+                goto out;
+        }
+        runinit (&runner);
+        snprintf (msg, sizeof (msg), "running lvs command, "
+                  "for getting snap status");
+        /* Using lvs command fetch the Volume Group name,
+         * Percentage of data filled and Logical Volume size
+         *
+         * "-o" argument is used to get the desired information,
+         * example : "lvs /dev/VolGroup/thin_vol -o vgname,lv_size",
+         * will get us Volume Group name and Logical Volume size.
+         *
+         * Here separator used is ":",
+         * for the above given command with separator ":",
+         * The output will be "vgname:lvsize"
+         */
+        runner_add_args (&runner, "lvs", device, "--noheading", "-o",
+                         "vg_name,data_percent,lv_size",
+                         "--separator", ":", NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        runner_log (&runner, "", GF_LOG_DEBUG, msg);
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not perform lvs action");
+                goto end;
+        }
+        do {
+                ptr = fgets (buf, sizeof (buf),
+                             runner_chio (&runner, STDOUT_FILENO));
+
+                if (ptr == NULL)
+                        break;
+                token = strtok (buf, ":");
+                if (token != NULL) {
+                        while (token && token[0] == ' ')
+                                token++;
+                        if (!token) {
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Invalid vg entry");
+                                goto end;
+                        }
+                        value = gf_strdup (token);
+                        if (!value) {
+                                ret = -1;
+                                goto end;
+                        }
+                        ret = snprintf (key, sizeof (key), "%s.vgname",
+                                        key_prefix);
+                        if (ret < 0) {
+                                goto end;
+                        }
+
+                        ret = dict_set_dynstr (rsp_dict, key, value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save vgname ");
+                                goto end;
+                        }
+                }
+
+                token = strtok (NULL, ":");
+                if (token != NULL) {
+                        value = gf_strdup (token);
+                        if (!value) {
+                                ret = -1;
+                                goto end;
+                        }
+                        ret = snprintf (key, sizeof (key), "%s.data",
+                                        key_prefix);
+                        if (ret < 0) {
+                                goto end;
+                        }
+
+                        ret = dict_set_dynstr (rsp_dict, key, value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save data percent ");
+                                goto end;
+                        }
+                }
+                token = strtok (NULL, ":");
+                if (token != NULL) {
+                        value = gf_strdup (token);
+                        if (!value) {
+                                ret = -1;
+                                goto end;
+                        }
+                        ret = snprintf (key, sizeof (key), "%s.lvsize",
+                                        key_prefix);
+                        if (ret < 0) {
+                                goto end;
+                        }
+
+                        ret = dict_set_dynstr (rsp_dict, key, value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not save meta data percent ");
+                                goto end;
+                        }
+                }
+
+        } while (ptr != NULL);
+
+        ret = 0;
+
+end:
+        runner_end (&runner);
+
+out:
+        if (ret && value) {
+                GF_FREE (value);
+        }
+
+        return ret;
+}
+
+int
+glusterd_get_single_brick_status (char **op_errstr, dict_t *rsp_dict,
+                                 char *keyprefix, int index,
+                                 glusterd_volinfo_t *snap_volinfo,
+                                 glusterd_brickinfo_t *brickinfo)
+{
+        int             ret                     = -1;
+        xlator_t        *this                   = NULL;
+        glusterd_conf_t *priv                   = NULL;
+        char            key[PATH_MAX]           = "";
+        char            *device                 = NULL;
+        char            *value                  = NULL;
+        char            brick_path[PATH_MAX]    = "";
+        char            pidfile[PATH_MAX]       = "";
+        pid_t           pid                     = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (keyprefix);
+        GF_ASSERT (snap_volinfo);
+        GF_ASSERT (brickinfo);
+
+        ret = snprintf (key, sizeof (key), "%s.brick%d.path", keyprefix,
+                        index);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = snprintf (brick_path, sizeof (brick_path),
+                        "%s:%s", brickinfo->hostname, brickinfo->path);
+        if (ret < 0) {
+                goto out;
+        }
+
+        value = gf_strdup (brick_path);
+        if (!value) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, key, value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to store "
+                        "brick_path %s", brickinfo->path);
+                goto out;
+        }
+
+        if (brickinfo->snap_status == -1) {
+                /* Setting vgname as "Pending Snapshot" */
+                value = gf_strdup ("Pending Snapshot");
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "%s.brick%d.vgname",
+                          keyprefix, index);
+                ret = dict_set_dynstr (rsp_dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save vgname ");
+                        goto out;
+                }
+
+                ret = 0;
+                goto out;
+        }
+        value = NULL;
+
+        ret = snprintf (key, sizeof (key), "%s.brick%d.status",
+                        keyprefix, index);
+        if (ret < 0) {
+                goto out;
+        }
+
+        if (brickinfo->status == GF_BRICK_STOPPED) {
+                value = gf_strdup ("No");
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (rsp_dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save brick status");
+                        goto out;
+                }
+                value = NULL;
+        } else {
+                value = gf_strdup ("Yes");
+                if (!value) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (rsp_dict, key, value);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save brick status");
+                        goto out;
+                }
+                value = NULL;
+
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_volinfo,
+                                            brickinfo, priv);
+                ret = glusterd_is_service_running (pidfile, &pid);
+
+                ret = snprintf (key, sizeof (key), "%s.brick%d.pid",
+                                keyprefix, index);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_set_int32 (rsp_dict, key, pid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save pid %d", pid);
+                        goto out;
+                }
+        }
+
+        ret = snprintf (key, sizeof (key), "%s.brick%d",
+                        keyprefix, index);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = glusterd_get_brick_lvm_details (rsp_dict, brickinfo,
+                                              snap_volinfo->volname,
+                                              device, key);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                        "brick LVM details");
+                goto out;
+        }
+out:
+        if (ret && value) {
+                GF_FREE (value);
+        }
+
+        return ret;
+}
+
+int
+glusterd_get_single_snap_status (char **op_errstr, dict_t *rsp_dict,
+                                 char *keyprefix, glusterd_snap_t *snap)
+{
+        int                      ret                 =       -1;
+        xlator_t                *this                =       NULL;
+        char                     key[PATH_MAX]       =       "";
+        char                     brickkey[PATH_MAX]  =       "";
+        glusterd_volinfo_t      *snap_volinfo        =       NULL;
+        glusterd_volinfo_t      *tmp_volinfo         =       NULL;
+        glusterd_brickinfo_t    *brickinfo           =       NULL;
+        int                      volcount            =       0;
+        int                      brickcount          =       0;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (keyprefix);
+        GF_ASSERT (snap);
+
+        list_for_each_entry_safe (snap_volinfo, tmp_volinfo, &snap->volumes,
+                                  vol_list) {
+                ret = snprintf (key, sizeof (key), "%s.vol%d", keyprefix,
+                                volcount);
+                if (ret < 0) {
+                        goto out;
+                }
+                list_for_each_entry (brickinfo, &snap_volinfo->bricks,
+                                     brick_list) {
+                        if (!glusterd_is_local_brick (this, snap_volinfo,
+                            brickinfo)) {
+                                brickcount++;
+                                continue;
+                        }
+
+                        ret = glusterd_get_single_brick_status (op_errstr,
+                                                  rsp_dict, key, brickcount,
+                                                  snap_volinfo, brickinfo);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Getting "
+                                        "single snap status failed");
+                                goto out;
+                        }
+                        brickcount++;
+                }
+                ret = snprintf (brickkey, sizeof (brickkey), "%s.brickcount",
+                                key);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = dict_set_int32 (rsp_dict, brickkey, brickcount);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not save brick count");
+                        goto out;
+                }
+                volcount++;
+        }
+
+        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_set_int32 (rsp_dict, key, volcount);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not save volcount");
+                goto out;
+        }
+
+out:
+
+        return ret;
+}
+
+int
+glusterd_get_each_snap_object_status (char **op_errstr, dict_t *rsp_dict,
+                                      glusterd_snap_t *snap, char *keyprefix)
+{
+        int                     ret             =       -1;
+        char                    key[PATH_MAX]   =       "";
+        char                    *temp           =       NULL;
+        xlator_t                *this           =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (snap);
+        GF_ASSERT (keyprefix);
+
+        /* TODO : Get all the snap volume info present in snap object,
+         * as of now, There will be only one snapvolinfo per snap object
+         */
+        ret = snprintf (key, sizeof (key), "%s.snapname", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        temp = gf_strdup (snap->snapname);
+        if (temp == NULL) {
+                ret = -1;
+                goto out;
+        }
+        ret = dict_set_dynstr (rsp_dict, key, temp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save "
+                        "snap name");
+                goto out;
+        }
+
+        temp = NULL;
+
+        ret = snprintf (key, sizeof (key), "%s.uuid", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        temp = gf_strdup (uuid_utoa (snap->snap_id));
+        if (temp == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (rsp_dict, key, temp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save "
+                        "snap UUID");
+                goto out;
+        }
+
+        temp = NULL;
+
+        ret = glusterd_get_single_snap_status (op_errstr, rsp_dict, keyprefix,
+                                               snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not get single snap status");
+                goto out;
+        }
+
+        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = dict_set_int32 (rsp_dict, key, 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save volcount");
+                goto out;
+        }
+out:
+        if (ret && temp)
+                GF_FREE (temp);
+
+        return ret;
+}
+
+int
+glusterd_get_snap_status_of_volume (char **op_errstr, dict_t *rsp_dict,
+                                    char *volname, char *keyprefix) {
+        int                     ret              =       -1;
+        glusterd_volinfo_t      *snap_volinfo    =       NULL;
+        glusterd_volinfo_t      *temp_volinfo    =       NULL;
+        glusterd_volinfo_t      *volinfo         =       NULL;
+        char                    key[PATH_MAX]    =        "";
+        xlator_t                *this            =       NULL;
+        glusterd_conf_t         *priv            =       NULL;
+        int                     i                =        0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (volname);
+        GF_ASSERT (keyprefix);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volinfo of "
+                        "volume %s", volname);
+                goto out;
+        }
+
+        list_for_each_entry_safe (snap_volinfo, temp_volinfo,
+                             &volinfo->snap_volumes, snapvol_list) {
+                ret = snprintf (key, sizeof (key), "status.snap%d", i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = glusterd_get_each_snap_object_status (op_errstr,
+                                       rsp_dict, snap_volinfo->snapshot, key);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Function : "
+                                "glusterd_get_single_snap_status failed");
+                        goto out;
+                }
+                i++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to save snapcount");
+                ret = -1;
+                goto out;
+        }
+out:
+        return ret;
+}
+
+int
+glusterd_get_all_snapshot_status (dict_t *dict, char **op_errstr,
+                              dict_t *rsp_dict)
+{
+        int32_t                 i               =       0;
+        int                     ret             =       -1;
+        char                    key[PATH_MAX]   =       "";
+        glusterd_conf_t         *priv           =       NULL;
+        glusterd_snap_t         *snap           =       NULL;
+        glusterd_snap_t         *tmp_snap       =       NULL;
+        xlator_t                *this           =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        list_for_each_entry_safe (snap, tmp_snap,
+                                  &priv->snapshots, snap_list) {
+                ret = snprintf (key, sizeof (key), "status.snap%d", i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = glusterd_get_each_snap_object_status (op_errstr,
+                                                         rsp_dict, snap, key);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not get "
+                                "the details of a snap object: %s",
+                                snap->snapname);
+                        goto out;
+                }
+                i++;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not save snapcount");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+glusterd_snapshot_status_commit (dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+        xlator_t        *this             =       NULL;
+        int             ret               =       -1;
+        glusterd_conf_t *conf             =       NULL;
+        char            *get_buffer       =       NULL;
+        int32_t         cmd               =       -1;
+        char            *snapname         =       NULL;
+        glusterd_snap_t *snap             =       NULL;
+        char            *volname          =       NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (op_errstr);
+
+        conf = this->private;
+
+        GF_ASSERT (conf);
+        ret = dict_get_int32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get status cmd type");
+                goto out;
+        }
+
+        ret = dict_set_int32 (rsp_dict, "cmd", cmd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not save status cmd in rsp dictionary");
+                goto out;
+        }
+        switch (cmd) {
+                case GF_SNAP_STATUS_TYPE_ALL:
+                {
+                        ret = glusterd_get_all_snapshot_status (dict, op_errstr,
+                                                           rsp_dict);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get snapshot status");
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_SNAP:
+                {
+
+                        ret = dict_get_str (dict, "snapname", &snapname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get snap name");
+                                goto out;
+                        }
+
+                        snap = glusterd_find_snap_by_name (snapname);
+                        if (!snap) {
+                                ret = gf_asprintf (op_errstr, "Snap (%s) "
+                                                  "not found", snapname);
+                                if (ret < 0) {
+                                        goto out;
+                                }
+                                ret = -1;
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get snap volinfo");
+                                goto out;
+                        }
+                        ret = glusterd_get_each_snap_object_status (op_errstr,
+                                          rsp_dict, snap, "status.snap0");
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                        "get status of snap %s", get_buffer);
+                                goto out;
+                        }
+                        break;
+                }
+                case GF_SNAP_STATUS_TYPE_VOL:
+                {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Unable to"
+                                        " get volume name");
+                                goto out;
+                        }
+
+                        ret = glusterd_get_snap_status_of_volume (op_errstr,
+                                           rsp_dict, volname, "status.vol0");
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Function :"
+                                        " glusterd_get_snap_status_of_volume "
+                                        "failed");
+                                goto out;
+                        }
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snapshot_create_postvalidate (dict_t *dict, int32_t op_ret,
+                                       char **op_errstr, dict_t *rsp_dict)
+{
+        xlator_t        *this           = NULL;
+        glusterd_conf_t *priv           = NULL;
+        int              ret            = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (op_ret) {
+                ret = glusterd_do_snap_cleanup (dict, op_errstr, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "cleanup operation "
+                                "failed");
+                        goto out;
+                }
+        } else {
+                ret = glusterd_snapshot_update_snaps_post_validate (dict,
+                                                                    op_errstr,
+                                                                    rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "create snapshot");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_snapshot (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+
+        xlator_t        *this           = NULL;
+        glusterd_conf_t *priv           = NULL;
+        int32_t          snap_command   = 0;
+        int              ret            = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case (GF_SNAP_OPTION_TYPE_CREATE):
+                ret = glusterd_snapshot_create_commit (dict, op_errstr,
+                                                       rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "create snapshot");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                ret = glusterd_snapshot_config_commit (dict, op_errstr,
+                                                       rsp_dict);
+                break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snapshot_remove_commit (dict, op_errstr,
+                                                       rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "delete snapshot");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_snapshot_restore (dict, op_errstr,
+                                                 rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Failed to "
+                                "restore snapshot");
+                        goto out;
+                }
+
+                break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = glusterd_snapshot_status_commit (dict, op_errstr,
+                                                       rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "show snapshot status");
+                        goto out;
+                }
+                break;
+
+
+        default:
+                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
+                goto out;
+                break;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_brickop (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int            ret       = -1;
+        int64_t        vol_count = 0;
+        int64_t        count     = 1;
+        char           key[1024] = {0,};
+        char           *volname  = NULL;
+        int32_t        snap_command = 0;
+        xlator_t       *this     = NULL;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = dict_get_int64 (dict, "volcount", &vol_count);
+                if (ret)
+                        goto out;
+                while (count <= vol_count) {
+                        snprintf (key, 1024, "volname%"PRId64, count);
+                        ret = dict_get_str (dict, key, &volname);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to get volname");
+                                goto out;
+                        }
+                        ret = dict_set_str (dict, "volname", volname);
+                        if (ret)
+                                goto out;
+
+                        ret = gd_brick_op_phase (GD_OP_SNAP, NULL, dict,
+                                        op_errstr);
+                        if (ret)
+                                goto out;
+                        volname = NULL;
+                        count++;
+                }
+
+                dict_del (dict, "volname");
+                ret = 0;
+                break;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                break;
+        default:
+                break;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_prevalidate (dict_t *dict, char **op_errstr,
+                               dict_t *rsp_dict)
+{
+        int                snap_command          = 0;
+        xlator_t           *this                 = NULL;
+        int                ret                   = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case (GF_SNAP_OPTION_TYPE_CREATE):
+                ret = glusterd_snapshot_create_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
+                                "pre-validation failed");
+                        goto out;
+                }
+                break;
+
+        case (GF_SNAP_OPTION_TYPE_CONFIG):
+                ret = glusterd_snapshot_config_prevalidate (dict, op_errstr);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot config "
+                                "pre-validation failed");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_snapshot_restore_prevalidate (dict, op_errstr,
+                                                             rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot restore "
+                                        "validation failed");
+                        goto out;
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snapshot_remove_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot remove "
+                                "validation failed");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = glusterd_snapshot_status_prevalidate (dict, op_errstr,
+                                                            rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot status "
+                                "validation failed");
+                        goto out;
+                }
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_snapshot_postvalidate (dict_t *dict, int32_t op_ret, char **op_errstr,
+                                dict_t *rsp_dict)
+{
+        int                snap_command          = 0;
+        xlator_t           *this                 = NULL;
+        int                ret                   = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (dict, "type", &snap_command);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = glusterd_snapshot_create_postvalidate (dict, op_ret,
+                                                             op_errstr,
+                                                             rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
+                                "post-validation failed");
+                        goto out;
+                }
+                break;
+
+        case GF_SNAP_OPTION_TYPE_DELETE:
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_snapshot_update_snaps_post_validate (dict,
+                                                                    op_errstr,
+                                                                    rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                "update missed snaps list");
+                        goto out;
+                }
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_handle_snapshot_fn (rpcsvc_request_t *req)
+{
+        int32_t               ret            = 0;
+        dict_t               *dict           = NULL;
+        gf_cli_req            cli_req        = {{0},};
+        glusterd_op_t         cli_op         = GD_OP_SNAP;
+        int                   type           = 0;
+        glusterd_conf_t      *conf           = NULL;
+        char                 *host_uuid      = NULL;
+        char                  err_str[2048]  = {0,};
+        xlator_t             *this           = NULL;
+        char                 *volname        = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req,
+                              (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len > 0) {
+                dict = dict_new ();
+                if (!dict)
+                        goto out;
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                }
+
+                dict->extra_stdfree = cli_req.dict.dict_val;
+
+                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
+                if (host_uuid == NULL) {
+                        snprintf (err_str, sizeof (err_str), "Failed to get "
+                                  "the uuid of local glusterd");
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
+                if (ret) {
+                        GF_FREE (host_uuid);
+                        goto out;
+                }
+
+
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "request dict length is %d",
+                        cli_req.dict.dict_len);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Command type not found");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        switch (type) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+                ret = glusterd_handle_snapshot_create (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
+                                        "failed: %s", err_str);
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+                ret = glusterd_handle_snapshot_restore (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot restore "
+                                        "failed: %s", err_str);
+                }
+
+                break;
+        case GF_SNAP_OPTION_TYPE_INFO:
+                ret = glusterd_handle_snapshot_info (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot info "
+                                        "failed");
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_LIST:
+                ret = glusterd_handle_snapshot_list (req, cli_op, dict,
+                                                err_str, sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot list "
+                                        "failed");
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                /* TODO : Type of lock to be taken when we are setting
+                 * limits system wide
+                 */
+                ret = dict_get_str (dict, "volname", &volname);
+                if (!volname) {
+                        ret = dict_set_int32 (dict, "hold_vol_locks",
+                                              _gf_false);
+                        if (ret) {
+                                gf_log ("cli", GF_LOG_ERROR,
+                                        "Unable to set hold_vol_locks value "
+                                        "as _gf_false");
+                                goto out;
+                        }
+
+                }
+                ret = glusterd_mgmt_v3_initiate_all_phases (req, cli_op, dict);
+                break;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_handle_snapshot_remove (req, cli_op, dict,
+                                                       err_str,
+                                                       sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot delete "
+                                "failed: %s", err_str);
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_START:
+        case GF_SNAP_OPTION_TYPE_STOP:
+        case GF_SNAP_OPTION_TYPE_STATUS:
+                ret = glusterd_handle_snapshot_status (req, cli_op, dict,
+                                                       err_str,
+                                                       sizeof (err_str));
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING, "Snapshot status "
+                                "failed: %s", err_str);
+                }
+                break;
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Unkown snapshot request "
+                        "type (%d)", type);
+                ret = -1; /* Failure */
+        }
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_snapshot (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req, glusterd_handle_snapshot_fn);
+}
+
+static inline void
+glusterd_free_snap_op (glusterd_snap_op_t *snap_op)
+{
+        if (snap_op) {
+                if (snap_op->brick_path)
+                        GF_FREE (snap_op->brick_path);
+
+                GF_FREE (snap_op);
+        }
+}
+
+/* Look for duplicates and accordingly update the list */
+int32_t
+glusterd_update_missed_snap_entry (glusterd_missed_snap_info *missed_snapinfo,
+                                   glusterd_snap_op_t *missed_snap_op)
+{
+        int32_t                        ret                         = -1;
+        glusterd_snap_op_t            *snap_opinfo                 = NULL;
+        gf_boolean_t                   match                       = _gf_false;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(missed_snapinfo);
+        GF_ASSERT(missed_snap_op);
+
+        list_for_each_entry (snap_opinfo, &missed_snapinfo->snap_ops,
+                             snap_ops_list) {
+                if ((!strcmp (snap_opinfo->brick_path,
+                              missed_snap_op->brick_path)) &&
+                    (snap_opinfo->op == missed_snap_op->op)) {
+                        /* If two entries have conflicting status
+                         * GD_MISSED_SNAP_DONE takes precedence
+                         */
+                        if ((snap_opinfo->status == GD_MISSED_SNAP_PENDING) &&
+                            (missed_snap_op->status == GD_MISSED_SNAP_DONE)) {
+                                snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "Updating missed snap status "
+                                        "for %s:%d:%s:%d as DONE",
+                                        missed_snapinfo->node_snap_info,
+                                        snap_opinfo->brick_num,
+                                        snap_opinfo->brick_path,
+                                        snap_opinfo->op);
+                                ret = 0;
+                                glusterd_free_snap_op (missed_snap_op);
+                                goto out;
+                        }
+                        match = _gf_true;
+                        break;
+                } else if ((snap_opinfo->brick_num ==
+                            missed_snap_op->brick_num) &&
+                            (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
+                            (missed_snap_op->op ==
+                             GF_SNAP_OPTION_TYPE_DELETE)) {
+                        /* Optimizing create and delete entries for the same
+                         * brick and same node
+                         */
+                        gf_log (this->name, GF_LOG_INFO,
+                                "Updating missed snap status "
+                                "for %s:%d:%s:%d as DONE",
+                                missed_snapinfo->node_snap_info,
+                                snap_opinfo->brick_num,
+                                snap_opinfo->brick_path,
+                                snap_opinfo->op);
+                        snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                        ret = 0;
+                        glusterd_free_snap_op (missed_snap_op);
+                        goto out;
+                }
+        }
+
+        if (match == _gf_true) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Duplicate entry. Not updating");
+                glusterd_free_snap_op (missed_snap_op);
+        } else {
+                list_add_tail (&missed_snap_op->snap_ops_list,
+                               &missed_snapinfo->snap_ops);
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Add new missed snap entry to the missed_snaps list. */
+int32_t
+glusterd_store_missed_snaps_list (char *missed_info, int32_t brick_num,
+                                  char *brick_path, int32_t snap_op,
+                                  int32_t snap_status)
+{
+        int32_t                        ret                         = -1;
+        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
+        glusterd_snap_op_t            *missed_snap_op              = NULL;
+        glusterd_conf_t               *priv                        = NULL;
+        gf_boolean_t                   match                       = _gf_false;
+        gf_boolean_t                   free_missed_snap_info       = _gf_false;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(missed_info);
+        GF_ASSERT(brick_path);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Create the snap_op object consisting of the *
+         * snap id and the op */
+        ret = glusterd_missed_snap_op_new (&missed_snap_op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create new missed snap object.");
+                ret = -1;
+                goto out;
+        }
+
+        missed_snap_op->brick_path = gf_strdup(brick_path);
+        if (!missed_snap_op->brick_path) {
+                ret = -1;
+                goto out;
+        }
+        missed_snap_op->brick_num = brick_num;
+        missed_snap_op->op = snap_op;
+        missed_snap_op->status = snap_status;
+
+        /* Look for other entries for the same node and same snap */
+        list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
+                             missed_snaps) {
+                if (!strcmp (missed_snapinfo->node_snap_info,
+                             missed_info)) {
+                        /* Found missed snapshot info for *
+                         * the same node and same snap */
+                        match = _gf_true;
+                        break;
+                }
+        }
+
+        if (match == _gf_false) {
+                /* First snap op missed for the brick */
+                ret = glusterd_missed_snapinfo_new (&missed_snapinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to create missed snapinfo");
+                        goto out;
+                }
+                free_missed_snap_info = _gf_true;
+                missed_snapinfo->node_snap_info = gf_strdup(missed_info);
+                if (!missed_snapinfo->node_snap_info) {
+                        ret = -1;
+                        goto out;
+                }
+
+                list_add_tail (&missed_snap_op->snap_ops_list,
+                               &missed_snapinfo->snap_ops);
+                list_add_tail (&missed_snapinfo->missed_snaps,
+                               &priv->missed_snaps_list);
+
+                ret = 0;
+                goto out;
+        } else {
+                ret = glusterd_update_missed_snap_entry (missed_snapinfo,
+                                                         missed_snap_op);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to update existing missed snap entry.");
+                        goto out;
+                }
+        }
+
+out:
+        if (ret) {
+                glusterd_free_snap_op (missed_snap_op);
+
+                if (missed_snapinfo &&
+                    (free_missed_snap_info == _gf_true)) {
+                        if (missed_snapinfo->node_snap_info)
+                                GF_FREE (missed_snapinfo->node_snap_info);
+
+                        GF_FREE (missed_snapinfo);
+                }
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Add  missing snap entries to the in-memory conf->missed_snap_list */
+int32_t
+glusterd_add_missed_snaps_to_list (dict_t *dict, int32_t missed_snap_count)
+{
+        char                          *buf                         = NULL;
+        char                          *tmp                         = NULL;
+        char                          *save_ptr                    = NULL;
+        char                          *nodeid                      = NULL;
+        char                          *snap_uuid                   = NULL;
+        char                          *brick_path                  = NULL;
+        char                           missed_info[PATH_MAX]       = "";
+        char                           name_buf[PATH_MAX]          = "";
+        int32_t                        i                           = -1;
+        int32_t                        ret                         = -1;
+        int32_t                        brick_num                   = -1;
+        int32_t                        snap_op                     = -1;
+        int32_t                        snap_status                 = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* We can update the missed_snaps_list without acquiring *
+         * any additional locks as big lock will be held.        */
+        for (i = 0; i < missed_snap_count; i++) {
+                snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                          i);
+                ret = dict_get_str (dict, name_buf, &buf);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to fetch %s", name_buf);
+                        goto out;
+                }
+
+                gf_log (this->name, GF_LOG_DEBUG, "missed_snap_entry = %s",
+                        buf);
+
+                /* Need to make a duplicate string coz the same dictionary *
+                 * is resent to the non-originator nodes */
+                tmp = gf_strdup (buf);
+                if (!tmp) {
+                        ret = -1;
+                        goto out;
+                }
+
+                /* Fetch the node-id, snap-id, brick_num,
+                 * brick_path, snap_op and snap status
+                 */
+                nodeid = strtok_r (tmp, ":", &save_ptr);
+                snap_uuid = strtok_r (NULL, "=", &save_ptr);
+                brick_num = atoi(strtok_r (NULL, ":", &save_ptr));
+                brick_path = strtok_r (NULL, ":", &save_ptr);
+                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
+                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
+
+                if (!nodeid || !snap_uuid || !brick_path ||
+                    brick_num < 1 || snap_op < 1 ||
+                    snap_status < 1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Invalid missed_snap_entry");
+                        ret = -1;
+                        goto out;
+                }
+
+                snprintf (missed_info, sizeof(missed_info), "%s:%s",
+                          nodeid, snap_uuid);
+
+                ret = glusterd_store_missed_snaps_list (missed_info,
+                                                        brick_num,
+                                                        brick_path,
+                                                        snap_op,
+                                                        snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store missed snaps_list");
+                        goto out;
+                }
+
+                GF_FREE (tmp);
+                tmp = NULL;
+        }
+
+out:
+        if (tmp)
+                GF_FREE (tmp);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
new file mode 100644
index 000000000..1c2ec58e8
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -0,0 +1,3564 @@
+/*
+   Copyright (c) 2007-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterd-op-sm.h"
+#include <inttypes.h>
+
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "compat.h"
+#include "dict.h"
+#include "protocol-common.h"
+#include "xlator.h"
+#include "logging.h"
+#include "timer.h"
+#include "defaults.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "statedump.h"
+#include "glusterd-mem-types.h"
+#include "glusterd.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-hooks.h"
+#include "store.h"
+#include "glusterd-store.h"
+
+#include "rpc-clnt.h"
+#include "common-utils.h"
+
+#include <sys/resource.h>
+#include <inttypes.h>
+#include <dirent.h>
+
+void
+glusterd_replace_slash_with_hyphen (char *str)
+{
+        char                    *ptr = NULL;
+
+        ptr = strchr (str, '/');
+
+        while (ptr) {
+                *ptr = '-';
+                ptr = strchr (str, '/');
+        }
+}
+
+int32_t
+glusterd_store_create_brick_dir (glusterd_volinfo_t *volinfo)
+{
+        int32_t                 ret = -1;
+        char                    brickdirpath[PATH_MAX] = {0,};
+        glusterd_conf_t         *priv = NULL;
+
+        GF_ASSERT (volinfo);
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        GLUSTERD_GET_BRICK_DIR (brickdirpath, volinfo, priv);
+        ret = gf_store_mkdir (brickdirpath);
+
+        return ret;
+}
+
+static void
+glusterd_store_key_vol_brick_set (glusterd_brickinfo_t *brickinfo,
+                                char *key_vol_brick, size_t len)
+{
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (key_vol_brick);
+        GF_ASSERT (len >= PATH_MAX);
+
+        snprintf (key_vol_brick, len, "%s", brickinfo->path);
+        glusterd_replace_slash_with_hyphen (key_vol_brick);
+}
+
+static void
+glusterd_store_brickinfofname_set (glusterd_brickinfo_t *brickinfo,
+                                char *brickfname, size_t len)
+{
+        char                    key_vol_brick[PATH_MAX] = {0};
+
+        GF_ASSERT (brickfname);
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (len >= PATH_MAX);
+
+        glusterd_store_key_vol_brick_set (brickinfo, key_vol_brick,
+                                        sizeof (key_vol_brick));
+        snprintf (brickfname, len, "%s:%s", brickinfo->hostname, key_vol_brick);
+}
+
+static void
+glusterd_store_brickinfopath_set (glusterd_volinfo_t *volinfo,
+                              glusterd_brickinfo_t *brickinfo,
+                              char *brickpath, size_t len)
+{
+        char                    brickfname[PATH_MAX] = {0};
+        char                    brickdirpath[PATH_MAX] = {0,};
+        glusterd_conf_t         *priv = NULL;
+
+        GF_ASSERT (brickpath);
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (len >= PATH_MAX);
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        GLUSTERD_GET_BRICK_DIR (brickdirpath, volinfo, priv);
+        glusterd_store_brickinfofname_set (brickinfo, brickfname,
+                                        sizeof (brickfname));
+        snprintf (brickpath, len, "%s/%s", brickdirpath, brickfname);
+}
+
+gf_boolean_t
+glusterd_store_is_valid_brickpath (char *volname, char *brick)
+{
+        char                    brickpath[PATH_MAX] = {0};
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        int32_t                 ret = 0;
+        size_t                  volname_len = strlen (volname);
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create brick "
+                        "info for brick %s", brick);
+                ret = 0;
+                goto out;
+        }
+        ret = glusterd_volinfo_new (&volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create volinfo");
+                ret = 0;
+                goto out;
+        }
+        if (volname_len >= sizeof (volinfo->volname)) {
+                gf_log (this->name, GF_LOG_WARNING, "volume name too long");
+                ret = 0;
+                goto out;
+        }
+        memcpy (volinfo->volname, volname, volname_len+1);
+        glusterd_store_brickinfopath_set (volinfo, brickinfo, brickpath,
+                                                sizeof (brickpath));
+
+        ret = (strlen (brickpath) < _POSIX_PATH_MAX);
+
+out:
+        if (brickinfo)
+                glusterd_brickinfo_delete (brickinfo);
+        if (volinfo)
+                glusterd_volinfo_delete (volinfo);
+
+        return ret;
+}
+
+int32_t
+glusterd_store_volinfo_brick_fname_write (int vol_fd,
+                                         glusterd_brickinfo_t *brickinfo,
+                                         int32_t brick_count)
+{
+        char                    key[PATH_MAX] = {0,};
+        char                    brickfname[PATH_MAX] = {0,};
+        int32_t                 ret = -1;
+
+        snprintf (key, sizeof (key), "%s-%d", GLUSTERD_STORE_KEY_VOL_BRICK,
+                  brick_count);
+        glusterd_store_brickinfofname_set (brickinfo, brickfname,
+                                        sizeof (brickfname));
+        ret = gf_store_save_value (vol_fd, key, brickfname);
+        if (ret)
+                goto out;
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_store_create_brick_shandle_on_absence (glusterd_volinfo_t *volinfo,
+                                                glusterd_brickinfo_t *brickinfo)
+{
+        char                    brickpath[PATH_MAX] = {0,};
+        int32_t                 ret = 0;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+
+        glusterd_store_brickinfopath_set (volinfo, brickinfo, brickpath,
+                                          sizeof (brickpath));
+        ret = gf_store_handle_create_on_absence (&brickinfo->shandle,
+                                                 brickpath);
+        return ret;
+}
+
+int32_t
+glusterd_store_brickinfo_write (int fd, glusterd_brickinfo_t *brickinfo)
+{
+        char                    value[256] = {0,};
+        int32_t                 ret = 0;
+
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (fd > 0);
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
+                                   brickinfo->hostname);
+        if (ret)
+                goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PATH,
+                                   brickinfo->path);
+        if (ret)
+                goto out;
+
+        snprintf (value, sizeof(value), "%d", brickinfo->port);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PORT, value);
+
+        snprintf (value, sizeof(value), "%d", brickinfo->rdma_port);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                   value);
+
+        snprintf (value, sizeof(value), "%d", brickinfo->decommissioned);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                                   value);
+        if (ret)
+                goto out;
+
+        if (strlen(brickinfo->device_path) > 0) {
+                snprintf (value, sizeof(value), "%s", brickinfo->device_path);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                                           value);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (value, sizeof(value), "%d", brickinfo->snap_status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                                   value);
+        if (ret)
+                goto out;
+
+        if (!brickinfo->vg[0])
+                goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                         brickinfo->vg);
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_perform_brick_store (glusterd_brickinfo_t *brickinfo)
+{
+        int                         fd = -1;
+        int32_t                     ret = -1;
+        GF_ASSERT (brickinfo);
+
+        fd = gf_store_mkstemp (brickinfo->shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_brickinfo_write (fd, brickinfo);
+        if (ret)
+                goto out;
+
+out:
+        if (ret && (fd > 0))
+                gf_store_unlink_tmppath (brickinfo->shandle);
+        if (fd > 0)
+                close (fd);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_brickinfo (glusterd_volinfo_t *volinfo,
+                          glusterd_brickinfo_t *brickinfo, int32_t brick_count,
+                          int vol_fd)
+{
+        int32_t                 ret = -1;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+
+        ret = glusterd_store_volinfo_brick_fname_write (vol_fd, brickinfo,
+                                                       brick_count);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_create_brick_dir (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_create_brick_shandle_on_absence (volinfo,
+                                                              brickinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_perform_brick_store (brickinfo);
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo, char *delete_path)
+{
+        int32_t                 ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        char                    brickpath[PATH_MAX] = {0,};
+        char                    *ptr = NULL;
+        char                    *tmppath = NULL;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (brickinfo);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        tmppath = gf_strdup (brickinfo->path);
+
+        ptr = strchr (tmppath, '/');
+
+        while (ptr) {
+                *ptr = '-';
+                ptr = strchr (tmppath, '/');
+        }
+
+        snprintf (brickpath, sizeof (brickpath),
+                  "%s/"GLUSTERD_BRICK_INFO_DIR"/%s:%s", delete_path,
+                  brickinfo->hostname, tmppath);
+
+        GF_FREE (tmppath);
+
+        ret = unlink (brickpath);
+
+        if ((ret < 0) && (errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_DEBUG, "Unlink failed on %s, "
+                        "reason: %s", brickpath, strerror(errno));
+                ret = -1;
+                goto out;
+        } else {
+                ret = 0;
+        }
+
+out:
+        if (brickinfo->shandle) {
+                gf_store_handle_destroy (brickinfo->shandle);
+                brickinfo->shandle = NULL;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo, char *delete_path)
+{
+        int32_t                 ret = 0;
+        glusterd_brickinfo_t    *tmp = NULL;
+        glusterd_conf_t         *priv = NULL;
+        char                    brickdir [PATH_MAX] = {0,};
+        DIR                     *dir = NULL;
+        struct dirent           *entry = NULL;
+        char                    path[PATH_MAX] = {0,};
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
+                ret = glusterd_store_delete_brick (tmp, delete_path);
+                if (ret)
+                        goto out;
+        }
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        snprintf (brickdir, sizeof (brickdir), "%s/%s", delete_path,
+                  GLUSTERD_BRICK_INFO_DIR);
+
+        dir = opendir (brickdir);
+
+        glusterd_for_each_entry (entry, dir);
+
+        while (entry) {
+                snprintf (path, sizeof (path), "%s/%s",
+                          brickdir, entry->d_name);
+                ret = unlink (path);
+                if (ret && errno != ENOENT) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Unable to unlink %s, "
+                                "reason: %s", path, strerror(errno));
+                }
+                glusterd_for_each_entry (entry, dir);
+        }
+
+        closedir (dir);
+
+        ret = rmdir (brickdir);
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+static int
+_storeslaves (dict_t *this, char *key, data_t *value, void *data)
+{
+        int32_t                      ret = 0;
+        gf_store_handle_t           *shandle = NULL;
+        xlator_t                    *xl = NULL;
+
+        xl = THIS;
+        GF_ASSERT (xl);
+
+        shandle = (gf_store_handle_t*)data;
+
+        GF_ASSERT (shandle);
+        GF_ASSERT (shandle->fd > 0);
+        GF_ASSERT (shandle->path);
+        GF_ASSERT (key);
+        GF_ASSERT (value && value->data);
+
+        if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
+                return -1;
+
+        if (!key)
+                return -1;
+        if (!value || !value->data)
+                return -1;
+
+        gf_log (xl->name, GF_LOG_DEBUG, "Storing in volinfo:key= %s, val=%s",
+                key, value->data);
+
+        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
+        if (ret) {
+                gf_log (xl->name, GF_LOG_ERROR, "Unable to write into store"
+                                " handle for path: %s", shandle->path);
+                return -1;
+        }
+        return 0;
+}
+
+
+int _storeopts (dict_t *this, char *key, data_t *value, void *data)
+{
+        int32_t                      ret = 0;
+        int32_t                      exists = 0;
+        gf_store_handle_t           *shandle = NULL;
+        xlator_t                    *xl = NULL;
+
+        xl = THIS;
+        GF_ASSERT (xl);
+
+        shandle = (gf_store_handle_t*)data;
+
+        GF_ASSERT (shandle);
+        GF_ASSERT (shandle->fd > 0);
+        GF_ASSERT (shandle->path);
+        GF_ASSERT (key);
+        GF_ASSERT (value && value->data);
+
+        if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
+                return -1;
+
+        if (!key)
+                return -1;
+        if (!value || !value->data)
+                return -1;
+
+        if (is_key_glusterd_hooks_friendly (key)) {
+                exists = 1;
+
+        } else {
+                exists = glusterd_check_option_exists (key, NULL);
+        }
+
+        if (1 == exists) {
+                gf_log (xl->name, GF_LOG_DEBUG, "Storing in volinfo:key= %s, "
+                        "val=%s", key, value->data);
+
+        } else {
+                gf_log (xl->name, GF_LOG_DEBUG, "Discarding:key= %s, val=%s",
+                        key, value->data);
+                return 0;
+        }
+
+        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
+        if (ret) {
+                gf_log (xl->name, GF_LOG_ERROR, "Unable to write into store"
+                        " handle for path: %s", shandle->path);
+                return -1;
+        }
+        return 0;
+}
+
+int32_t
+glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
+{
+        char    *str            = NULL;
+        char    buf[PATH_MAX]   = {0,};
+        int32_t ret             = -1;
+
+        GF_ASSERT (fd > 0);
+        GF_ASSERT (volinfo);
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->type);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TYPE, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->brick_count);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_COUNT, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STATUS, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->sub_count);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_SUB_COUNT, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->stripe_count);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->replica_count);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
+                                   buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->version);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_VERSION, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->transport_type);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TRANSPORT, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%s", volinfo->parent_volname);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PARENT_VOLNAME, buf);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to store "
+                        GLUSTERD_STORE_KEY_PARENT_VOLNAME);
+                goto out;
+        }
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_ID,
+                                   uuid_utoa (volinfo->volume_id));
+        if (ret)
+                goto out;
+
+        str = glusterd_auth_get_username (volinfo);
+        if (str) {
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_USERNAME,
+                                           str);
+                if (ret)
+                goto out;
+        }
+
+        str = glusterd_auth_get_password (volinfo);
+        if (str) {
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PASSWORD,
+                                           str);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->op_version);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_OP_VERSION, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->client_op_version);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                                   buf);
+        if (ret)
+                goto out;
+        if (volinfo->caps) {
+                snprintf (buf, sizeof (buf), "%d", volinfo->caps);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CAPS,
+                                           buf);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->is_volume_restored);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_IS_RESTORED, buf);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to write is_volume_restored");
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%"PRIu64, volinfo->snap_max_hard_limit);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                                   buf);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Unable to write snap-max-hard-limit");
+                goto out;
+        }
+
+out:
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to write volume "
+                        "values for %s", volinfo->volname);
+        return ret;
+}
+
+static void
+glusterd_store_voldirpath_set (glusterd_volinfo_t *volinfo, char *voldirpath,
+                               size_t len)
+{
+        glusterd_conf_t         *priv = NULL;
+
+        GF_ASSERT (volinfo);
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        GLUSTERD_GET_VOLUME_DIR (voldirpath, volinfo, priv);
+}
+
+static int32_t
+glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)
+{
+        int32_t                 ret = -1;
+        char                    voldirpath[PATH_MAX] = {0,};
+
+        GF_ASSERT (volinfo);
+
+        glusterd_store_voldirpath_set (volinfo, voldirpath,
+                                       sizeof (voldirpath));
+        ret = gf_store_mkdir (voldirpath);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+static int32_t
+glusterd_store_create_snap_dir (glusterd_snap_t *snap)
+{
+        int32_t            ret                   = -1;
+        char               snapdirpath[PATH_MAX] = {0,};
+        glusterd_conf_t   *priv                  = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+
+        GLUSTERD_GET_SNAP_DIR (snapdirpath, snap, priv);
+
+        ret = mkdir_p (snapdirpath, 0755, _gf_true);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snaps dir "
+                        "%s", snapdirpath);
+        }
+        return ret;
+}
+
+int32_t
+glusterd_store_volinfo_write (int fd, glusterd_volinfo_t *volinfo)
+{
+        int32_t                         ret = -1;
+        gf_store_handle_t              *shandle = NULL;
+        GF_ASSERT (fd > 0);
+        GF_ASSERT (volinfo);
+        GF_ASSERT (volinfo->shandle);
+
+        shandle = volinfo->shandle;
+        ret = glusterd_volume_exclude_options_write (fd, volinfo);
+        if (ret)
+                goto out;
+
+        shandle->fd = fd;
+        dict_foreach (volinfo->dict, _storeopts, shandle);
+
+        dict_foreach (volinfo->gsync_slaves, _storeslaves, shandle);
+        shandle->fd = 0;
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_snapinfo_write (glusterd_snap_t *snap)
+{
+        int32_t  ret            = -1;
+        int      fd             = 0;
+        char     buf[PATH_MAX]  = "";
+
+        GF_ASSERT (snap);
+
+        fd = gf_store_mkstemp (snap->shandle);
+        if (fd <= 0)
+               goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_ID,
+                                   uuid_utoa (snap->snap_id));
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", snap->snap_status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_STATUS, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", snap->snap_restored);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_RESTORED, buf);
+        if (ret)
+                goto out;
+
+        if (snap->description) {
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_DESC,
+                                           snap->description);
+                if (ret)
+                        goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%ld", snap->time_stamp);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP, buf);
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static void
+glusterd_store_rbstatepath_set (glusterd_volinfo_t *volinfo, char *rbstatepath,
+                                size_t len)
+{
+        char    voldirpath[PATH_MAX] = {0,};
+        GF_ASSERT (volinfo);
+        GF_ASSERT (rbstatepath);
+        GF_ASSERT (len <= PATH_MAX);
+
+        glusterd_store_voldirpath_set (volinfo, voldirpath,
+                                       sizeof (voldirpath));
+        snprintf (rbstatepath, len, "%s/%s", voldirpath,
+                  GLUSTERD_VOLUME_RBSTATE_FILE);
+}
+
+static void
+glusterd_store_volfpath_set (glusterd_volinfo_t *volinfo, char *volfpath,
+                             size_t len)
+{
+        char    voldirpath[PATH_MAX] = {0,};
+        GF_ASSERT (volinfo);
+        GF_ASSERT (volfpath);
+        GF_ASSERT (len <= PATH_MAX);
+
+        glusterd_store_voldirpath_set (volinfo, voldirpath,
+                                       sizeof (voldirpath));
+        snprintf (volfpath, len, "%s/%s", voldirpath, GLUSTERD_VOLUME_INFO_FILE);
+}
+
+static void
+glusterd_store_node_state_path_set (glusterd_volinfo_t *volinfo,
+                                    char *node_statepath, size_t len)
+{
+        char    voldirpath[PATH_MAX] = {0,};
+        GF_ASSERT (volinfo);
+        GF_ASSERT (node_statepath);
+        GF_ASSERT (len <= PATH_MAX);
+
+        glusterd_store_voldirpath_set (volinfo, voldirpath,
+                                       sizeof (voldirpath));
+        snprintf (node_statepath, len, "%s/%s", voldirpath,
+                  GLUSTERD_NODE_STATE_FILE);
+}
+
+static void
+glusterd_store_missed_snaps_list_path_set (char *missed_snaps_list,
+                                           size_t len)
+{
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (missed_snaps_list);
+        GF_ASSERT (len <= PATH_MAX);
+
+        snprintf (missed_snaps_list, len, "%s/snaps/"
+                  GLUSTERD_MISSED_SNAPS_LIST_FILE, priv->workdir);
+}
+
+static void
+glusterd_store_snapfpath_set (glusterd_snap_t *snap, char *snap_fpath,
+                              size_t len)
+{
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+        GF_ASSERT (snap_fpath);
+        GF_ASSERT (len <= PATH_MAX);
+
+        snprintf (snap_fpath, len, "%s/snaps/%s/%s", priv->workdir,
+                  snap->snapname, GLUSTERD_SNAP_INFO_FILE);
+}
+
+int32_t
+glusterd_store_create_rbstate_shandle_on_absence (glusterd_volinfo_t *volinfo)
+{
+        char            rbstatepath[PATH_MAX] = {0};
+        int32_t         ret                   = 0;
+
+        GF_ASSERT (volinfo);
+
+        glusterd_store_rbstatepath_set (volinfo, rbstatepath, sizeof (rbstatepath));
+        ret = gf_store_handle_create_on_absence (&volinfo->rb_shandle,
+                                                 rbstatepath);
+        return ret;
+}
+
+int32_t
+glusterd_store_create_vol_shandle_on_absence (glusterd_volinfo_t *volinfo)
+{
+        char            volfpath[PATH_MAX] = {0};
+        int32_t         ret = 0;
+
+        GF_ASSERT (volinfo);
+
+        glusterd_store_volfpath_set (volinfo, volfpath, sizeof (volfpath));
+        ret = gf_store_handle_create_on_absence (&volinfo->shandle, volfpath);
+        return ret;
+}
+
+int32_t
+glusterd_store_create_nodestate_sh_on_absence (glusterd_volinfo_t *volinfo)
+{
+        char            node_state_path[PATH_MAX] = {0};
+        int32_t         ret                   = 0;
+
+        GF_ASSERT (volinfo);
+
+        glusterd_store_node_state_path_set (volinfo, node_state_path,
+                                            sizeof (node_state_path));
+        ret =
+          gf_store_handle_create_on_absence (&volinfo->node_state_shandle,
+                                             node_state_path);
+
+        return ret;
+}
+
+static int32_t
+glusterd_store_create_missed_snaps_list_shandle_on_absence ()
+{
+        char               missed_snaps_list[PATH_MAX] = "";
+        int32_t            ret                         = -1;
+        glusterd_conf_t   *priv                        = NULL;
+        xlator_t          *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        glusterd_store_missed_snaps_list_path_set (missed_snaps_list,
+                                                   sizeof(missed_snaps_list));
+
+        ret = gf_store_handle_create_on_absence
+                                            (&priv->missed_snaps_list_shandle,
+                                             missed_snaps_list);
+        return ret;
+}
+
+int32_t
+glusterd_store_create_snap_shandle_on_absence (glusterd_snap_t *snap)
+{
+        char            snapfpath[PATH_MAX] = {0};
+        int32_t         ret = 0;
+
+        GF_ASSERT (snap);
+
+        glusterd_store_snapfpath_set (snap, snapfpath, sizeof (snapfpath));
+        ret = gf_store_handle_create_on_absence (&snap->shandle, snapfpath);
+        return ret;
+}
+
+int32_t
+glusterd_store_brickinfos (glusterd_volinfo_t *volinfo, int vol_fd)
+{
+        int32_t                 ret = 0;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        int32_t                 brick_count = 0;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_store_brickinfo (volinfo, brickinfo,
+                                            brick_count, vol_fd);
+                if (ret)
+                        goto out;
+                brick_count++;
+        }
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_rbstate_write (int fd, glusterd_volinfo_t *volinfo)
+{
+        int     ret             = -1;
+        int     port            = 0;
+        char    buf[PATH_MAX]   = {0, };
+
+        GF_ASSERT (fd > 0);
+        GF_ASSERT (volinfo);
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->rep_brick.rb_status);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_STATUS, buf);
+        if (ret)
+                goto out;
+
+        if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
+
+                snprintf (buf, sizeof (buf), "%s:%s",
+                          volinfo->rep_brick.src_brick->hostname,
+                          volinfo->rep_brick.src_brick->path);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                                           buf);
+                if (ret)
+                        goto out;
+
+                snprintf (buf, sizeof (buf), "%s:%s",
+                          volinfo->rep_brick.dst_brick->hostname,
+                          volinfo->rep_brick.dst_brick->path);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                                           buf);
+                if (ret)
+                        goto out;
+
+                switch (volinfo->transport_type) {
+                        case GF_TRANSPORT_RDMA:
+                                port = volinfo->rep_brick.dst_brick->rdma_port;
+                                break;
+
+                        case GF_TRANSPORT_TCP:
+                        case GF_TRANSPORT_BOTH_TCP_RDMA:
+                                port = volinfo->rep_brick.dst_brick->port;
+                                break;
+                }
+
+                snprintf (buf, sizeof (buf), "%d", port);
+                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_PORT,
+                                           buf);
+                if (ret)
+                        goto out;
+                uuid_unparse (volinfo->rep_brick.rb_id, buf);
+                ret = gf_store_save_value (fd, GF_REPLACE_BRICK_TID_KEY, buf);
+        }
+
+        ret = 0;
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_perform_rbstate_store (glusterd_volinfo_t *volinfo)
+{
+        int                         fd = -1;
+        int32_t                     ret = -1;
+        GF_ASSERT (volinfo);
+
+        fd = gf_store_mkstemp (volinfo->rb_shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_rbstate_write (fd, volinfo);
+        if (ret)
+                goto out;
+
+        ret = gf_store_rename_tmppath (volinfo->rb_shandle);
+        if (ret)
+                goto out;
+
+out:
+        if (ret && (fd > 0))
+                gf_store_unlink_tmppath (volinfo->rb_shandle);
+        if (fd > 0)
+                close (fd);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
+{
+        int     ret             = -1;
+        char    buf[PATH_MAX]   = {0, };
+
+        GF_ASSERT (fd > 0);
+        GF_ASSERT (volinfo);
+
+        if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_STATUS) {
+                ret = 0;
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.defrag_cmd);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_DEFRAG, buf);
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.op);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_DEFRAG_OP, buf);
+        if (ret)
+                goto out;
+
+        if (volinfo->rebal.defrag_cmd) {
+                uuid_unparse (volinfo->rebal.rebalance_id, buf);
+                ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf);
+        }
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo)
+{
+        int                         fd = -1;
+        int32_t                     ret = -1;
+        GF_ASSERT (volinfo);
+
+        fd = gf_store_mkstemp (volinfo->node_state_shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_node_state_write (fd, volinfo);
+        if (ret)
+                goto out;
+
+        ret = gf_store_rename_tmppath (volinfo->node_state_shandle);
+        if (ret)
+                goto out;
+
+out:
+        if (ret && (fd > 0))
+                gf_store_unlink_tmppath (volinfo->node_state_shandle);
+        if (fd > 0)
+                close (fd);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo)
+{
+        int                         fd = -1;
+        int32_t                     ret = -1;
+        GF_ASSERT (volinfo);
+
+        fd = gf_store_mkstemp (volinfo->shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo_write (fd, volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_brickinfos (volinfo, fd);
+        if (ret)
+                goto out;
+
+out:
+        if (ret && (fd > 0))
+                gf_store_unlink_tmppath (volinfo->shandle);
+        if (fd > 0)
+                close (fd);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+void
+glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
+                                         glusterd_volinfo_ver_ac_t ac)
+{
+        GF_ASSERT (volinfo);
+
+        switch (ac) {
+        case GLUSTERD_VOLINFO_VER_AC_NONE:
+        break;
+        case GLUSTERD_VOLINFO_VER_AC_INCREMENT:
+                volinfo->version++;
+        break;
+        case GLUSTERD_VOLINFO_VER_AC_DECREMENT:
+                volinfo->version--;
+        break;
+        }
+}
+
+void
+glusterd_store_bricks_cleanup_tmp (glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t    *brickinfo           = NULL;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                gf_store_unlink_tmppath (brickinfo->shandle);
+        }
+}
+
+void
+glusterd_store_volume_cleanup_tmp (glusterd_volinfo_t *volinfo)
+{
+        GF_ASSERT (volinfo);
+
+        glusterd_store_bricks_cleanup_tmp (volinfo);
+
+        gf_store_unlink_tmppath (volinfo->shandle);
+
+        gf_store_unlink_tmppath (volinfo->rb_shandle);
+
+        gf_store_unlink_tmppath (volinfo->node_state_shandle);
+}
+
+void
+glusterd_store_snap_cleanup_tmp (glusterd_snap_t *snap)
+{
+        GF_ASSERT (snap);
+
+        gf_store_unlink_tmppath (snap->shandle);
+}
+
+int32_t
+glusterd_store_brickinfos_atomic_update (glusterd_volinfo_t *volinfo)
+{
+        int                      ret            = -1;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = gf_store_rename_tmppath (brickinfo->shandle);
+                if (ret)
+                        goto out;
+        }
+out:
+        return ret;
+}
+
+int32_t
+glusterd_store_volinfo_atomic_update (glusterd_volinfo_t *volinfo)
+{
+        int ret = -1;
+        GF_ASSERT (volinfo);
+
+        ret = gf_store_rename_tmppath (volinfo->shandle);
+        if (ret)
+                goto out;
+
+out:
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't rename "
+                        "temporary file(s): Reason %s", strerror (errno));
+        return ret;
+}
+
+int32_t
+glusterd_store_volume_atomic_update (glusterd_volinfo_t *volinfo)
+{
+        int ret = -1;
+        GF_ASSERT (volinfo);
+
+        ret = glusterd_store_brickinfos_atomic_update (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_volinfo_atomic_update (volinfo);
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_store_snap_atomic_update (glusterd_snap_t *snap)
+{
+        int ret = -1;
+        GF_ASSERT (snap);
+
+        ret = gf_store_rename_tmppath (snap->shandle);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't rename "
+                        "temporary file(s): Reason %s", strerror (errno));
+
+        return ret;
+}
+
+int32_t
+glusterd_store_snap (glusterd_snap_t *snap)
+{
+        int32_t                 ret = -1;
+
+        GF_ASSERT (snap);
+
+        ret = glusterd_store_create_snap_dir (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snap dir");
+                goto out;
+        }
+
+        ret = glusterd_store_create_snap_shandle_on_absence (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snap info "
+                        "file");
+                goto out;
+        }
+
+        ret = glusterd_store_snapinfo_write (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to write snap info");
+                goto out;
+        }
+
+        ret = glusterd_store_snap_atomic_update (snap);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,"Failed to do automic update");
+                goto out;
+        }
+
+out:
+        if (ret)
+                glusterd_store_snap_cleanup_tmp (snap);
+
+        gf_log (THIS->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac)
+{
+        int32_t                 ret = -1;
+
+        GF_ASSERT (volinfo);
+
+        glusterd_perform_volinfo_version_action (volinfo, ac);
+        ret = glusterd_store_create_volume_dir (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_create_vol_shandle_on_absence (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_create_rbstate_shandle_on_absence (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_create_nodestate_sh_on_absence (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_perform_volume_store (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_volume_atomic_update (volinfo);
+        if (ret) {
+                glusterd_perform_volinfo_version_action (volinfo,
+                                                         GLUSTERD_VOLINFO_VER_AC_DECREMENT);
+                goto out;
+        }
+
+        ret = glusterd_store_perform_rbstate_store (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_perform_node_state_store (volinfo);
+        if (ret)
+                goto out;
+
+        //checksum should be computed at the end
+        ret = glusterd_volume_compute_cksum (volinfo);
+        if (ret)
+                goto out;
+
+out:
+        if (ret)
+                glusterd_store_volume_cleanup_tmp (volinfo);
+
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_store_delete_volume (glusterd_volinfo_t *volinfo)
+{
+        char             pathname[PATH_MAX]    = {0,};
+        int32_t          ret                   = 0;
+        glusterd_conf_t *priv                  = NULL;
+        DIR             *dir                   = NULL;
+        struct dirent   *entry                 = NULL;
+        char             path[PATH_MAX]        = {0,};
+        char             delete_path[PATH_MAX] = {0,};
+        char             trashdir[PATH_MAX]    = {0,};
+        struct stat      st                    = {0, };
+        xlator_t        *this                  = NULL;
+        gf_boolean_t     rename_fail           = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (volinfo);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
+
+        snprintf (delete_path, sizeof (delete_path),
+        "%s/"GLUSTERD_TRASH"/%s.deleted", priv->workdir,
+        uuid_utoa (volinfo->volume_id));
+
+        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
+                  priv->workdir);
+
+        ret = mkdir (trashdir, 0777);
+        if (ret && errno != EEXIST) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create trash "
+                        "directory, reason : %s", strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = rename (pathname, delete_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to rename volume "
+                        "directory for volume %s", volinfo->volname);
+                rename_fail = _gf_true;
+                goto out;
+        }
+
+        dir = opendir (delete_path);
+        if (!dir) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to open directory %s."
+                        " Reason : %s", delete_path, strerror (errno));
+                ret = 0;
+                goto out;
+        }
+        ret = glusterd_store_remove_bricks (volinfo, delete_path);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Remove bricks failed for %s",
+                        volinfo->volname);
+        }
+
+        glusterd_for_each_entry (entry, dir);
+        while (entry) {
+
+                snprintf (path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
+                ret = stat (path, &st);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Failed to stat "
+                                "entry %s : %s", path, strerror (errno));
+                        goto stat_failed;
+                }
+
+                if (S_ISDIR (st.st_mode))
+                        ret = rmdir (path);
+                else
+                        ret = unlink (path);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, " Failed to remove "
+                                "%s. Reason : %s", path, strerror (errno));
+                }
+
+                gf_log (this->name, GF_LOG_DEBUG, "%s %s",
+                                ret ? "Failed to remove":"Removed",
+                                entry->d_name);
+stat_failed:
+                memset (path, 0, sizeof(path));
+                glusterd_for_each_entry (entry, dir);
+        }
+
+        ret = closedir (dir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to close dir %s. "
+                        "Reason : %s",delete_path, strerror (errno));
+        }
+
+        ret = rmdir (delete_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s,err: %s",
+                        delete_path, strerror (errno));
+        }
+        ret = rmdir (trashdir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s, Reason:"
+                        " %s", trashdir, strerror (errno));
+        }
+
+out:
+        if (volinfo->shandle) {
+                gf_store_handle_destroy (volinfo->shandle);
+                volinfo->shandle = NULL;
+        }
+        ret = (rename_fail == _gf_true) ? -1: 0;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/*TODO: cleanup the duplicate code and implement a generic function for
+ * deleting snap/volume depending on the parameter flag */
+int32_t
+glusterd_store_delete_snap (glusterd_snap_t *snap)
+{
+        char             pathname[PATH_MAX]    = {0,};
+        int32_t          ret                   = 0;
+        glusterd_conf_t *priv                  = NULL;
+        DIR             *dir                   = NULL;
+        struct dirent   *entry                 = NULL;
+        char             path[PATH_MAX]        = {0,};
+        char             delete_path[PATH_MAX] = {0,};
+        char             trashdir[PATH_MAX]    = {0,};
+        struct stat      st                    = {0, };
+        xlator_t        *this                  = NULL;
+        gf_boolean_t     rename_fail           = _gf_false;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (snap);
+        GLUSTERD_GET_SNAP_DIR (pathname, snap, priv);
+
+        snprintf (delete_path, sizeof (delete_path),
+        "%s/"GLUSTERD_TRASH"/snap-%s.deleted", priv->workdir,
+        uuid_utoa (snap->snap_id));
+
+        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
+                  priv->workdir);
+
+        ret = mkdir (trashdir, 0777);
+        if (ret && errno != EEXIST) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create trash "
+                        "directory, reason : %s", strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = rename (pathname, delete_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to rename snap "
+                        "directory %s to %s", snap->snapname, delete_path);
+                rename_fail = _gf_true;
+                goto out;
+        }
+
+        dir = opendir (delete_path);
+        if (!dir) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to open directory %s."
+                        " Reason : %s", delete_path, strerror (errno));
+                ret = 0;
+                goto out;
+        }
+
+        glusterd_for_each_entry (entry, dir);
+        while (entry) {
+                snprintf (path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
+                ret = stat (path, &st);
+                if (ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Failed to stat "
+                                "entry %s : %s", path, strerror (errno));
+                        goto stat_failed;
+                }
+
+                if (S_ISDIR (st.st_mode))
+                        ret = rmdir (path);
+                else
+                        ret = unlink (path);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, " Failed to remove "
+                                "%s. Reason : %s", path, strerror (errno));
+                }
+
+                gf_log (this->name, GF_LOG_DEBUG, "%s %s",
+                                ret ? "Failed to remove":"Removed",
+                                entry->d_name);
+stat_failed:
+                memset (path, 0, sizeof(path));
+                glusterd_for_each_entry (entry, dir);
+        }
+
+        ret = closedir (dir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to close dir %s. "
+                        "Reason : %s",delete_path, strerror (errno));
+        }
+
+        ret = rmdir (delete_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s,err: %s",
+                        delete_path, strerror (errno));
+        }
+        ret = rmdir (trashdir);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s, Reason:"
+                        " %s", trashdir, strerror (errno));
+        }
+
+out:
+        if (snap->shandle) {
+                gf_store_handle_destroy (snap->shandle);
+                snap->shandle = NULL;
+        }
+        ret = (rename_fail == _gf_true) ? -1: 0;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_store_global_info (xlator_t *this)
+{
+        int                     ret                     = -1;
+        glusterd_conf_t         *conf                   = NULL;
+        char                    op_version_str[15]      = {0,};
+        char                    path[PATH_MAX]          = {0,};
+        gf_store_handle_t       *handle                 = NULL;
+        char                    *uuid_str               = NULL;
+        char                     buf[256]               = {0, };
+
+        conf = this->private;
+
+        uuid_str = gf_strdup (uuid_utoa (MY_UUID));
+        if (!uuid_str)
+                goto out;
+
+        if (!conf->handle) {
+                snprintf (path, PATH_MAX, "%s/%s", conf->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_new (path, &handle);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to get store handle");
+                        goto out;
+                }
+
+                conf->handle = handle;
+        } else
+                handle = conf->handle;
+
+        /* These options need to be available for all users */
+        ret = chmod (handle->path, 0644);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "chmod error for %s: %s",
+                        GLUSTERD_INFO_FILE, strerror (errno));
+                goto out;
+        }
+
+        handle->fd = gf_store_mkstemp (handle);
+        if (handle->fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = gf_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY,
+                                   uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Storing uuid failed ret = %d", ret);
+                goto out;
+        }
+
+        snprintf (op_version_str, 15, "%d", conf->op_version);
+        ret = gf_store_save_value (handle->fd, GD_OP_VERSION_KEY,
+                                   op_version_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Storing op-version failed ret = %d", ret);
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%"PRIu64, conf->snap_max_hard_limit);
+        ret = gf_store_save_value (handle->fd,
+                                   GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT, buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Storing snap-max-hard-limit failed ret = %d", ret);
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%"PRIu64, conf->snap_max_soft_limit);
+        ret = gf_store_save_value (handle->fd,
+                                   GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT, buf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Storing snap-max-soft-limit failed ret = %d", ret);
+                goto out;
+        }
+
+        ret = gf_store_rename_tmppath (handle);
+out:
+        if (ret && (handle->fd > 0))
+                gf_store_unlink_tmppath (handle);
+
+        if (handle->fd > 0) {
+                close (handle->fd);
+                handle->fd = 0;
+        }
+
+        if (uuid_str)
+                GF_FREE (uuid_str);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to store glusterd global-info");
+
+        return ret;
+}
+
+int
+glusterd_retrieve_op_version (xlator_t *this, int *op_version)
+{
+        char                    *op_version_str = NULL;
+        glusterd_conf_t         *priv           = NULL;
+        int                     ret             = -1;
+        int                     tmp_version     = 0;
+        char                    *tmp            = NULL;
+        char                    path[PATH_MAX]  = {0,};
+        gf_store_handle_t       *handle         = NULL;
+
+        priv = this->private;
+
+        if (!priv->handle) {
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_retrieve (path, &handle);
+
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Unable to get store "
+                                "handle!");
+                        goto out;
+                }
+
+                priv->handle = handle;
+        }
+
+        ret = gf_store_retrieve_value (priv->handle, GD_OP_VERSION_KEY,
+                                       &op_version_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No previous op_version present");
+                goto out;
+        }
+
+        tmp_version = strtol (op_version_str, &tmp, 10);
+        if ((tmp_version <= 0) || (tmp && strlen (tmp) > 1)) {
+                gf_log (this->name, GF_LOG_WARNING, "invalid version number");
+                goto out;
+        }
+
+        *op_version = tmp_version;
+
+        ret = 0;
+out:
+        if (op_version_str)
+                GF_FREE (op_version_str);
+
+        return ret;
+}
+
+int
+glusterd_retrieve_sys_snap_max_limit (xlator_t *this, uint64_t *limit,
+                                      char *key)
+{
+        char                    *limit_str      = NULL;
+        glusterd_conf_t         *priv           = NULL;
+        int                     ret             = -1;
+        uint64_t                tmp_limit       = 0;
+        char                    *tmp            = NULL;
+        char                    path[PATH_MAX]  = {0,};
+        gf_store_handle_t       *handle         = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+        GF_ASSERT (limit);
+        GF_ASSERT (key);
+
+        if (!priv->handle) {
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_retrieve (path, &handle);
+
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Unable to get store "
+                                "handle!");
+                        goto out;
+                }
+
+                priv->handle = handle;
+        }
+
+        ret = gf_store_retrieve_value (priv->handle,
+                                       key,
+                                       &limit_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "No previous %s present", key);
+                goto out;
+        }
+
+        tmp_limit = strtoul (limit_str, &tmp, 10);
+        if ((tmp_limit <= 0) || (tmp && strlen (tmp) > 1)) {
+                gf_log (this->name, GF_LOG_WARNING, "invalid version number");
+                goto out;
+        }
+
+        *limit = tmp_limit;
+
+        ret = 0;
+out:
+        if (limit_str)
+                GF_FREE (limit_str);
+
+        return ret;
+}
+static int
+glusterd_restore_op_version (xlator_t *this)
+{
+        glusterd_conf_t *conf           = NULL;
+        int              ret            = 0;
+        int              op_version     = 0;
+
+        conf = this->private;
+
+        ret = glusterd_retrieve_sys_snap_max_limit (this,
+                                                    &conf->snap_max_hard_limit,
+                                                    GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unable to retrieve system snap-max-hard-limit, "
+                        "setting it to default value(%d)",
+                        GLUSTERD_SNAPS_MAX_HARD_LIMIT);
+                conf->snap_max_hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+        }
+
+        ret = glusterd_retrieve_sys_snap_max_limit (this,
+                                                    &conf->snap_max_soft_limit,
+                                                    GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unable to retrieve system snap-max-soft-limit, "
+                        "setting it to default value(%d)",
+                        GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT);
+                conf->snap_max_soft_limit = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+        }
+
+        ret = glusterd_retrieve_op_version (this, &op_version);
+        if (!ret) {
+                if ((op_version < GD_OP_VERSION_MIN) ||
+                    (op_version > GD_OP_VERSION_MAX)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "wrong op-version (%d) retrieved", op_version);
+                        ret = -1;
+                        goto out;
+                }
+                conf->op_version = op_version;
+                gf_log ("glusterd", GF_LOG_INFO,
+                        "retrieved op-version: %d", conf->op_version);
+                goto out;
+        }
+
+        /* op-version can be missing from the store file in 2 cases,
+         * 1. This is a new install of glusterfs
+         * 2. This is an upgrade of glusterfs from a version without op-version
+         *    to a version with op-version (eg. 3.3 -> 3.4)
+         *
+         * Detection of a new install or an upgrade from an older install can be
+         * done by checking for the presence of the its peer-id in the store
+         * file.  If peer-id is present, the installation is an upgrade else, it
+         * is a new install.
+         *
+         * For case 1, set op-version to GD_OP_VERSION_MAX.
+         * For case 2, set op-version to GD_OP_VERSION_MIN.
+         */
+        ret = glusterd_retrieve_uuid();
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO, "Detected new install. Setting"
+                        " op-version to maximum : %d", GD_OP_VERSION_MAX);
+                conf->op_version = GD_OP_VERSION_MAX;
+        } else {
+                gf_log (this->name, GF_LOG_INFO, "Upgrade detected. Setting"
+                        " op-version to minimum : %d", GD_OP_VERSION_MIN);
+                conf->op_version = GD_OP_VERSION_MIN;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_retrieve_uuid ()
+{
+        char            *uuid_str = NULL;
+        int32_t         ret = -1;
+        gf_store_handle_t *handle = NULL;
+        glusterd_conf_t *priv = NULL;
+        char            path[PATH_MAX] = {0,};
+
+        priv = THIS->private;
+
+        if (!priv->handle) {
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_INFO_FILE);
+                ret = gf_store_handle_retrieve (path, &handle);
+
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Unable to get store"
+                                "handle!");
+                        goto out;
+                }
+
+                priv->handle = handle;
+        }
+
+        ret = gf_store_retrieve_value (priv->handle, GLUSTERD_STORE_UUID_KEY,
+                                       &uuid_str);
+
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "No previous uuid is present");
+                goto out;
+        }
+
+        uuid_parse (uuid_str, priv->uuid);
+
+out:
+        GF_FREE (uuid_str);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+int32_t
+glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
+{
+        int32_t                 ret = 0;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        gf_store_iter_t         *iter = NULL;
+        char                    *key = NULL;
+        char                    *value = NULL;
+        char                    brickdir[PATH_MAX] = {0,};
+        char                    path[PATH_MAX] = {0,};
+        glusterd_conf_t         *priv = NULL;
+        int32_t                 brick_count = 0;
+        char                    tmpkey[4096] = {0,};
+        gf_store_iter_t         *tmpiter = NULL;
+        char                    *tmpvalue = NULL;
+        struct pmap_registry    *pmap = NULL;
+        gf_store_op_errno_t     op_errno = GD_STORE_SUCCESS;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (volinfo->volname);
+
+        priv = THIS->private;
+
+        GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv);
+
+        ret = gf_store_iter_new (volinfo->shandle, &tmpiter);
+        if (ret)
+                goto out;
+
+        while (brick_count < volinfo->brick_count) {
+                ret = glusterd_brickinfo_new (&brickinfo);
+
+                if (ret)
+                        goto out;
+                snprintf (tmpkey, sizeof (tmpkey), "%s-%d",
+                          GLUSTERD_STORE_KEY_VOL_BRICK,brick_count);
+                ret = gf_store_iter_get_matching (tmpiter, tmpkey, &tmpvalue);
+                snprintf (path, sizeof (path), "%s/%s", brickdir, tmpvalue);
+
+                GF_FREE (tmpvalue);
+
+                tmpvalue = NULL;
+
+                ret = gf_store_handle_retrieve (path, &brickinfo->shandle);
+
+                if (ret)
+                        goto out;
+
+                ret = gf_store_iter_new (brickinfo->shandle, &iter);
+
+                if (ret)
+                        goto out;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Unable to iterate "
+                                "the store for brick: %s, reason: %s", path,
+                                gf_store_strerror (op_errno));
+                        goto out;
+                }
+                while (!ret) {
+                        if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
+                                      strlen (GLUSTERD_STORE_KEY_BRICK_HOSTNAME))) {
+                                strncpy (brickinfo->hostname, value, 1024);
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_PATH,
+                                    strlen (GLUSTERD_STORE_KEY_BRICK_PATH))) {
+                                strncpy (brickinfo->path, value,
+                                         sizeof (brickinfo->path));
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_PORT,
+                                    strlen (GLUSTERD_STORE_KEY_BRICK_PORT))) {
+                                gf_string2int (value, &brickinfo->port);
+
+                                if (brickinfo->port < priv->base_port) {
+                                        /* This is required to adhere to the
+                                           IANA standards */
+                                        brickinfo->port = 0;
+                                } else {
+                                        /* This is required to have proper ports
+                                           assigned to bricks after restart */
+                                        pmap = pmap_registry_get (THIS);
+                                        if (pmap->last_alloc <= brickinfo->port)
+                                                pmap->last_alloc =
+                                                        brickinfo->port + 1;
+                                }
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                    strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
+                                gf_string2int (value, &brickinfo->rdma_port);
+
+                                if (brickinfo->rdma_port < priv->base_port) {
+                                        /* This is required to adhere to the
+                                           IANA standards */
+                                        brickinfo->rdma_port = 0;
+                                } else {
+                                        /* This is required to have proper ports
+                                           assigned to bricks after restart */
+                                        pmap = pmap_registry_get (THIS);
+                                        if (pmap->last_alloc <=
+                                            brickinfo->rdma_port)
+                                                pmap->last_alloc =
+                                                        brickinfo->rdma_port +1;
+                                }
+
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                                             strlen (GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
+                                gf_string2int (value, &brickinfo->decommissioned);
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                                             strlen (GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH))) {
+                                strncpy (brickinfo->device_path, value,
+                                         sizeof (brickinfo->device_path));
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                                             strlen (GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
+                                gf_string2int (value, &brickinfo->snap_status);
+                        } else if (!strncmp (key,
+                                    GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                    strlen (GLUSTERD_STORE_KEY_BRICK_VGNAME))) {
+                                strncpy (brickinfo->vg, value,
+                                         sizeof (brickinfo->vg));
+                        } else {
+                                gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
+                                        key);
+                        }
+
+                        GF_FREE (key);
+                        GF_FREE (value);
+                        key = NULL;
+                        value = NULL;
+
+                        ret = gf_store_iter_get_next (iter, &key, &value,
+                                                      &op_errno);
+                }
+
+                if (op_errno != GD_STORE_EOF)
+                        goto out;
+                ret = gf_store_iter_destroy (iter);
+
+                if (ret)
+                        goto out;
+
+                list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
+                brick_count++;
+        }
+
+        ret = gf_store_iter_destroy (tmpiter);
+        if (ret)
+                goto out;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+
+int32_t
+glusterd_store_retrieve_rbstate (glusterd_volinfo_t *volinfo)
+{
+        int32_t                   ret                   = -1;
+        gf_store_iter_t           *iter                 = NULL;
+        char                      *key                  = NULL;
+        char                      *value                = NULL;
+        char                      volpath[PATH_MAX]     = {0,};
+        glusterd_conf_t           *priv                 = NULL;
+        char                      path[PATH_MAX]        = {0,};
+        gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
+        xlator_t                 *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volinfo);
+
+        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
+        snprintf (path, sizeof (path), "%s/%s", volpath,
+                  GLUSTERD_VOLUME_RBSTATE_FILE);
+
+        ret = gf_store_handle_retrieve (path, &volinfo->rb_shandle);
+
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_new (volinfo->rb_shandle, &iter);
+
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret)
+                goto out;
+
+        while (!ret) {
+                if (!strncmp (key, GLUSTERD_STORE_KEY_RB_STATUS,
+                              strlen (GLUSTERD_STORE_KEY_RB_STATUS))) {
+                        volinfo->rep_brick.rb_status = atoi (value);
+                }
+
+                if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
+                        if (!strncmp (key, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                                      strlen (GLUSTERD_STORE_KEY_RB_SRC_BRICK))) {
+                                ret = glusterd_brickinfo_new_from_brick (value,
+                                                &volinfo->rep_brick.src_brick);
+                                if (ret)
+                                        goto out;
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                                             strlen (GLUSTERD_STORE_KEY_RB_DST_BRICK))) {
+                                ret = glusterd_brickinfo_new_from_brick (value,
+                                                &volinfo->rep_brick.dst_brick);
+                                if (ret)
+                                        goto out;
+                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_PORT,
+                                             strlen (GLUSTERD_STORE_KEY_RB_DST_PORT))) {
+                                switch (volinfo->transport_type) {
+                                case GF_TRANSPORT_RDMA:
+                                volinfo->rep_brick.dst_brick->rdma_port =
+                                                 atoi (value);
+                                        break;
+
+                                case GF_TRANSPORT_TCP:
+                                case GF_TRANSPORT_BOTH_TCP_RDMA:
+                                volinfo->rep_brick.dst_brick->port =
+                                                atoi (value);
+                                        break;
+                                }
+                        } else if (!strncmp (key, GF_REPLACE_BRICK_TID_KEY,
+                                             strlen (GF_REPLACE_BRICK_TID_KEY))) {
+                                        uuid_parse (value,
+                                                    volinfo->rep_brick.rb_id);
+                        }
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+
+        ret = gf_store_iter_destroy (iter);
+
+        if (ret)
+                goto out;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_node_state (glusterd_volinfo_t *volinfo)
+{
+        int32_t                   ret                   = -1;
+        gf_store_iter_t           *iter                 = NULL;
+        char                      *key                  = NULL;
+        char                      *value                = NULL;
+        char                      volpath[PATH_MAX]     = {0,};
+        glusterd_conf_t           *priv                 = NULL;
+        char                      path[PATH_MAX]        = {0,};
+        gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
+        xlator_t                 *this                  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volinfo);
+
+        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
+        snprintf (path, sizeof (path), "%s/%s", volpath,
+                  GLUSTERD_NODE_STATE_FILE);
+
+        ret = gf_store_handle_retrieve (path, &volinfo->node_state_shandle);
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_new (volinfo->node_state_shandle, &iter);
+
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret)
+                goto out;
+
+        while (ret == 0) {
+                if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
+                              strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
+                        volinfo->rebal.defrag_cmd = atoi (value);
+                }
+
+                if (volinfo->rebal.defrag_cmd) {
+                        if (!strncmp (key, GF_REBALANCE_TID_KEY,
+                                       strlen (GF_REBALANCE_TID_KEY)))
+                                uuid_parse (value, volinfo->rebal.rebalance_id);
+
+                        if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP,
+                                      strlen (GLUSTERD_STORE_KEY_DEFRAG_OP)))
+                                volinfo->rebal.op = atoi (value);
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+
+        ret = gf_store_iter_destroy (iter);
+
+        if (ret)
+                goto out;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+
+        return ret;
+}
+
+
+int
+glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
+{
+        int                     ret = -1;
+        int                     exists                  = 0;
+        char                    *key                    = NULL;
+        char                    *value                  = NULL;
+        char                    volpath[PATH_MAX]       = {0,};
+        char                    path[PATH_MAX]          = {0,};
+        xlator_t                *this                   = NULL;
+        glusterd_conf_t         *conf                   = NULL;
+        gf_store_iter_t         *iter                   = NULL;
+        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = THIS->private;
+        GF_ASSERT (volinfo);
+
+        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
+
+        snprintf (path, sizeof (path), "%s/%s", volpath,
+                  GLUSTERD_VOLUME_INFO_FILE);
+
+        ret = gf_store_handle_retrieve (path, &volinfo->shandle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "volinfo handle is NULL");
+                goto out;
+        }
+
+        ret = gf_store_iter_new (volinfo->shandle, &iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get new store "
+                        "iter");
+                goto out;
+        }
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get next store "
+                        "iter");
+                goto out;
+        }
+
+        while (!ret) {
+                gf_log ("", GF_LOG_DEBUG, "key = %s value = %s", key, value);
+                if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_TYPE,
+                              strlen (GLUSTERD_STORE_KEY_VOL_TYPE))) {
+                        volinfo->type = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_COUNT,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_COUNT))) {
+                        volinfo->brick_count = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_STATUS,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_STATUS))) {
+                        volinfo->status = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_VERSION,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_VERSION))) {
+                        volinfo->version = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_PORT,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_PORT))) {
+                        volinfo->port = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_SUB_COUNT,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_SUB_COUNT))) {
+                        volinfo->sub_count = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_STRIPE_CNT))) {
+                        volinfo->stripe_count = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_REPLICA_CNT))) {
+                        volinfo->replica_count = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_TRANSPORT))) {
+                        volinfo->transport_type = atoi (value);
+                        volinfo->nfs_transport_type = volinfo->transport_type;
+                        if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) {
+                                volinfo->nfs_transport_type = GF_DEFAULT_NFS_TRANSPORT;
+                        }
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_ID,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_ID))) {
+                        ret = uuid_parse (value, volinfo->volume_id);
+                        if (ret)
+                                gf_log ("", GF_LOG_WARNING,
+                                        "failed to parse uuid");
+
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_USERNAME,
+                                     strlen (GLUSTERD_STORE_KEY_USERNAME))) {
+
+                        glusterd_auth_set_username (volinfo, value);
+
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_PASSWORD,
+                                     strlen (GLUSTERD_STORE_KEY_PASSWORD))) {
+
+                        glusterd_auth_set_password (volinfo, value);
+
+                } else if (strstr (key, "slave")) {
+                        ret = dict_set_dynstr (volinfo->gsync_slaves, key,
+                                                gf_strdup (value));
+                        if (ret) {
+                                gf_log ("",GF_LOG_ERROR, "Error in "
+                                                "dict_set_str");
+                                goto out;
+                        }
+                        gf_log ("", GF_LOG_DEBUG, "Parsed as "GEOREP" "
+                                " slave:key=%s,value:%s", key, value);
+
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_OP_VERSION,
+                                strlen (GLUSTERD_STORE_KEY_VOL_OP_VERSION))) {
+                        volinfo->op_version = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                                strlen (GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION))) {
+                        volinfo->client_op_version = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CAPS,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_CAPS))) {
+                        volinfo->caps = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                                strlen (GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT))) {
+                        volinfo->snap_max_hard_limit = (uint64_t) atoll (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_IS_RESTORED,
+                                     strlen (GLUSTERD_STORE_KEY_VOL_IS_RESTORED))) {
+                        volinfo->is_volume_restored = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
+                                strlen (GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
+                        strncpy (volinfo->parent_volname, value, sizeof(volinfo->parent_volname) - 1);
+                } else {
+
+                        if (is_key_glusterd_hooks_friendly (key)) {
+                                exists = 1;
+
+                        } else  {
+                                exists = glusterd_check_option_exists (key,
+                                                                       NULL);
+                        }
+
+                        switch (exists) {
+                        case -1:
+                                ret = -1;
+                                goto out;
+
+                        case 0:
+                                gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
+                                        key);
+                                break;
+
+                        case 1:
+                                ret = dict_set_str(volinfo->dict, key,
+                                                   gf_strdup (value));
+                                if (ret) {
+                                        gf_log ("",GF_LOG_ERROR, "Error in "
+                                                "dict_set_str");
+                                        goto out;
+                                }
+                                gf_log ("", GF_LOG_DEBUG, "Parsed as Volume-"
+                                        "set:key=%s,value:%s", key, value);
+                                break;
+                        }
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+
+        /* backward compatibility */
+        {
+
+                switch (volinfo->type) {
+
+                        case GF_CLUSTER_TYPE_NONE:
+                                volinfo->stripe_count  = 1;
+                                volinfo->replica_count = 1;
+                        break;
+
+                        case GF_CLUSTER_TYPE_STRIPE:
+                                volinfo->stripe_count  = volinfo->sub_count;
+                                volinfo->replica_count = 1;
+                        break;
+
+                        case GF_CLUSTER_TYPE_REPLICATE:
+                                volinfo->stripe_count  = 1;
+                                volinfo->replica_count = volinfo->sub_count;
+                        break;
+
+                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                                /* Introduced in 3.3 */
+                                GF_ASSERT (volinfo->stripe_count > 0);
+                                GF_ASSERT (volinfo->replica_count > 0);
+                        break;
+
+                        default:
+                                GF_ASSERT (0);
+                        break;
+                }
+
+                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+
+                volinfo->subvol_count = (volinfo->brick_count /
+                                         volinfo->dist_leaf_count);
+
+                /* Only calculate volume op-versions if they are not found */
+                if (!volinfo->op_version && !volinfo->client_op_version)
+                        gd_update_volume_op_versions (volinfo);
+        }
+
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+
+        ret = gf_store_iter_destroy (iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to destroy store "
+                        "iter");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+glusterd_volinfo_t*
+glusterd_store_retrieve_volume (char *volname, glusterd_snap_t *snap)
+{
+        int32_t                    ret                  = -1;
+        glusterd_volinfo_t        *volinfo              = NULL;
+        glusterd_volinfo_t        *origin_volinfo       = NULL;
+        glusterd_conf_t           *priv                 = NULL;
+        xlator_t                  *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volname);
+
+        ret = glusterd_volinfo_new (&volinfo);
+        if (ret)
+                goto out;
+
+        priv = THIS->private;
+
+        strncpy (volinfo->volname, volname, GLUSTERD_MAX_VOLUME_NAME);
+        volinfo->snapshot = snap;
+        if (snap)
+                volinfo->is_snap_volume = _gf_true;
+
+        ret = glusterd_store_update_volinfo (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to update volinfo "
+                        "for %s volume", volname);
+                goto out;
+        }
+
+        ret = glusterd_store_retrieve_bricks (volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_volume_compute_cksum (volinfo);
+        if (ret)
+                goto out;
+
+        if (!snap) {
+                list_add_tail (&volinfo->vol_list, &priv->volumes);
+        } else {
+                ret = glusterd_volinfo_find (volinfo->parent_volname,
+                                             &origin_volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Parent volinfo "
+                                "not found for %s volume", volname);
+                        goto out;
+                }
+                glusterd_list_add_snapvol (origin_volinfo, volinfo);
+        }
+out:
+        if (ret) {
+                if (volinfo)
+                        glusterd_volinfo_delete (volinfo);
+                volinfo = NULL;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+
+        return volinfo;
+}
+
+inline void
+glusterd_store_set_options_path (glusterd_conf_t *conf, char *path, size_t len)
+{
+        snprintf (path, len, "%s/options", conf->workdir);
+}
+
+int
+_store_global_opts (dict_t *this, char *key, data_t *value, void *data)
+{
+        gf_store_handle_t *shandle = data;
+
+        gf_store_save_value (shandle->fd, key, (char*)value->data);
+        return 0;
+}
+
+int32_t
+glusterd_store_options (xlator_t *this, dict_t *opts)
+{
+        gf_store_handle_t               *shandle = NULL;
+        glusterd_conf_t                 *conf = NULL;
+        char                            path[PATH_MAX] = {0};
+        int                             fd = -1;
+        int32_t                         ret = -1;
+
+        conf = this->private;
+        glusterd_store_set_options_path (conf, path, sizeof (path));
+
+        ret = gf_store_handle_new (path, &shandle);
+        if (ret)
+                goto out;
+
+        fd = gf_store_mkstemp (shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        shandle->fd = fd;
+        dict_foreach (opts, _store_global_opts, shandle);
+        shandle->fd = 0;
+        ret = gf_store_rename_tmppath (shandle);
+        if (ret)
+                goto out;
+out:
+        gf_store_handle_destroy (shandle);
+        if (fd >=0 )
+                close (fd);
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_options (xlator_t *this)
+{
+        char                            path[PATH_MAX] = {0};
+        glusterd_conf_t                 *conf = NULL;
+        gf_store_handle_t               *shandle = NULL;
+        gf_store_iter_t                 *iter = NULL;
+        char                            *key = NULL;
+        char                            *value = NULL;
+        gf_store_op_errno_t             op_errno = 0;
+        int                             ret = -1;
+
+        conf = this->private;
+        glusterd_store_set_options_path (conf, path, sizeof (path));
+
+        ret = gf_store_handle_retrieve (path, &shandle);
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_new (shandle, &iter);
+        if (ret)
+                goto out;
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        while (!ret) {
+                ret = dict_set_dynstr (conf->opts, key, value);
+                if (ret) {
+                        GF_FREE (key);
+                        GF_FREE (value);
+                        goto out;
+                }
+                GF_FREE (key);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+        ret = 0;
+out:
+        gf_store_iter_destroy (iter);
+        gf_store_handle_destroy (shandle);
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_volumes (xlator_t  *this, glusterd_snap_t *snap)
+{
+        int32_t                ret              = -1;
+        char                   path[PATH_MAX]   = {0,};
+        glusterd_conf_t       *priv             = NULL;
+        DIR                   *dir              = NULL;
+        struct dirent         *entry            = NULL;
+        glusterd_volinfo_t    *volinfo          = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        if (snap)
+                snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,
+                          snap->snapname);
+        else
+                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                          GLUSTERD_VOLUME_DIR_PREFIX);
+
+        dir = opendir (path);
+
+        if (!dir) {
+                gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
+                goto out;
+        }
+
+        glusterd_for_each_entry (entry, dir);
+
+        while (entry) {
+                if ( entry->d_type != DT_DIR )
+                        goto next;
+
+                volinfo = glusterd_store_retrieve_volume (entry->d_name, snap);
+                if (!volinfo) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to restore "
+                                "volume: %s", entry->d_name);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_store_retrieve_rbstate (volinfo);
+                if (ret) {
+                        /* Backward compatibility */
+                        gf_log ("", GF_LOG_INFO, "Creating a new rbstate "
+                                "for volume: %s.", entry->d_name);
+                        ret = glusterd_store_create_rbstate_shandle_on_absence (volinfo);
+                        ret = glusterd_store_perform_rbstate_store (volinfo);
+                }
+
+                ret = glusterd_store_retrieve_node_state (volinfo);
+                if (ret) {
+                        /* Backward compatibility */
+                        gf_log ("", GF_LOG_INFO, "Creating a new node_state "
+                                "for volume: %s.", entry->d_name);
+                        glusterd_store_create_nodestate_sh_on_absence (volinfo);
+                        ret = glusterd_store_perform_node_state_store (volinfo);
+
+                }
+
+next:
+                glusterd_for_each_entry (entry, dir);
+        }
+
+        ret = 0;
+out:
+        if (dir)
+                closedir (dir);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_resolve_snap_bricks (xlator_t  *this, glusterd_snap_t *snap)
+{
+        int32_t                 ret = -1;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+
+        GF_ASSERT (this);
+        GF_VALIDATE_OR_GOTO (this->name, snap, out);
+
+        list_for_each_entry (volinfo, &snap->volumes, vol_list) {
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        ret = glusterd_resolve_brick (brickinfo);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "resolve brick failed in restore");
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_store_update_snap (glusterd_snap_t *snap)
+{
+        int                     ret = -1;
+        char                    *key                    = NULL;
+        char                    *value                  = NULL;
+        char                    snappath[PATH_MAX]      = {0,};
+        char                    path[PATH_MAX]          = {0,};
+        xlator_t                *this                   = NULL;
+        glusterd_conf_t         *conf                   = NULL;
+        gf_store_iter_t         *iter                   = NULL;
+        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
+
+        this = THIS;
+        conf = this->private;
+        GF_ASSERT (snap);
+
+        GLUSTERD_GET_SNAP_DIR (snappath, snap, conf);
+
+        snprintf (path, sizeof (path), "%s/%s", snappath,
+                  GLUSTERD_SNAP_INFO_FILE);
+
+        ret = gf_store_handle_retrieve (path, &snap->shandle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "snap handle is NULL");
+                goto out;
+        }
+
+        ret = gf_store_iter_new (snap->shandle, &iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get new store "
+                        "iter");
+                goto out;
+        }
+
+        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get next store "
+                        "iter");
+                goto out;
+        }
+
+        while (!ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "key = %s value = %s",
+                        key, value);
+
+                if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_ID,
+                                     strlen (GLUSTERD_STORE_KEY_SNAP_ID))) {
+                        ret = uuid_parse (value, snap->snap_id);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Failed to parse uuid");
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_RESTORED,
+                                   strlen (GLUSTERD_STORE_KEY_SNAP_RESTORED))) {
+                        snap->snap_restored = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_STATUS,
+                                     strlen (GLUSTERD_STORE_KEY_SNAP_STATUS))) {
+                        snap->snap_status = atoi (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_DESC,
+                                     strlen (GLUSTERD_STORE_KEY_SNAP_DESC))) {
+                        snap->description = gf_strdup (value);
+                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP,
+                                  strlen (GLUSTERD_STORE_KEY_SNAP_TIMESTAMP))) {
+                        snap->time_stamp = atoi (value);
+                }
+
+                GF_FREE (key);
+                GF_FREE (value);
+                key = NULL;
+                value = NULL;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+        }
+
+        if (op_errno != GD_STORE_EOF)
+                goto out;
+
+        ret = gf_store_iter_destroy (iter);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to destroy store "
+                        "iter");
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_snap (char *snapname)
+{
+        int32_t                ret    = -1;
+        dict_t                *dict   = NULL;
+        glusterd_snap_t       *snap   = NULL;
+        glusterd_conf_t       *priv   = NULL;
+        xlator_t              *this   = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snapname);
+
+        dict = dict_new();
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create dict");
+                ret = -1;
+                goto out;
+        }
+
+        snap = glusterd_new_snap_object ();
+        if (!snap) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create "
+                                                  " snap object");
+                goto out;
+        }
+
+        strncpy (snap->snapname, snapname, strlen(snapname));
+        ret = glusterd_store_update_snap (snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to update snapshot "
+                        "for %s snap", snapname);
+                goto out;
+        }
+
+        ret = glusterd_store_retrieve_volumes (this, snap);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to retrieve "
+                        "snap volumes for snap %s", snapname);
+                goto out;
+        }
+
+        /* Unlike bricks of normal volumes which are resolved at the end of
+           the glusterd restore, the bricks belonging to the snap volumes of
+           each snap should be resolved as part of snapshot restore itself.
+           Because if the snapshot has to be removed, then resolving bricks
+           helps glusterd in understanding what all bricks have its own uuid
+           and killing those bricks.
+        */
+        ret = glusterd_resolve_snap_bricks (this, snap);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING, "resolving the snap bricks"
+                        " failed (snap: %s)", snap?snap->snapname:"");
+
+        /* When the snapshot command from cli is received, the on disk and
+           in memory structures for the snapshot are created (with the status)
+           being marked as GD_SNAP_STATUS_INIT. Once the backend snapshot is
+           taken, the status is changed to GD_SNAP_STATUS_IN_USE. If glusterd
+           dies after taking the backend snapshot, but before updating the
+           status, then when glusterd comes up, it should treat that snapshot
+           as a failed snapshot and clean it up.
+        */
+        if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
+                ret = glusterd_snap_remove (dict, snap, _gf_true, _gf_true);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING, "failed to remove"
+                                " the snapshot %s", snap->snapname);
+                goto out;
+        }
+
+        /* TODO: list_add_order can do 'N-square' comparisions and
+           is not efficient. Find a better solution to store the snap
+           in order */
+        list_add_order (&snap->snap_list, &priv->snapshots,
+                        glusterd_compare_snap_time);
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+/* Read the missed_snap_list and update the in-memory structs */
+int32_t
+glusterd_store_retrieve_missed_snaps_list (xlator_t  *this)
+{
+        char                   buf[PATH_MAX]    = "";
+        char                   path[PATH_MAX]   = "";
+        char                  *missed_node_info = NULL;
+        char                  *brick_path       = NULL;
+        char                  *value            = NULL;
+        char                  *save_ptr         = NULL;
+        FILE                  *fp               = NULL;
+        int32_t                brick_num        = -1;
+        int32_t                snap_op          = -1;
+        int32_t                snap_status      = -1;
+        int32_t                ret              = -1;
+        glusterd_conf_t       *priv             = NULL;
+        gf_store_op_errno_t    store_errno      = GD_STORE_SUCCESS;
+
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Get the path of the missed_snap_list */
+        glusterd_store_missed_snaps_list_path_set (path, sizeof(path));
+
+        fp = fopen (path, "r");
+        if (!fp) {
+                /* If errno is ENOENT then there are no missed snaps yet */
+                if (errno != ENOENT) {
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to open %s. "
+                                "Error: %s", path, strerror(errno));
+                } else {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "No missed snaps list.");
+                        ret = 0;
+                }
+                goto out;
+        }
+
+        do {
+                ret = gf_store_read_and_tokenize (fp, buf,
+                                                  &missed_node_info, &value,
+                                                  &store_errno);
+                if (ret) {
+                        if (store_errno == GD_STORE_EOF) {
+                                gf_log (this->name,
+                                        GF_LOG_DEBUG,
+                                        "EOF for missed_snap_list");
+                                ret = 0;
+                                break;
+                        }
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to fetch data from "
+                                "missed_snaps_list. Error: %s",
+                                gf_store_strerror (store_errno));
+                        goto out;
+                }
+
+                /* Fetch the brick_num, brick_path, snap_op and snap status */
+                brick_num = atoi(strtok_r (value, ":", &save_ptr));
+                brick_path = strtok_r (NULL, ":", &save_ptr);
+                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
+                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
+
+                if (!missed_node_info || !brick_path ||
+                    brick_num < 1 || snap_op < 1 ||
+                    snap_status < 1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Invalid missed_snap_entry");
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_store_missed_snaps_list (missed_node_info,
+                                                        brick_num,
+                                                        brick_path,
+                                                        snap_op,
+                                                        snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to store missed snaps_list");
+                        goto out;
+                }
+
+        } while (store_errno == GD_STORE_SUCCESS);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_snaps (xlator_t  *this)
+{
+        int32_t                ret              = 0;
+        char                   path[PATH_MAX]   = {0,};
+        glusterd_conf_t       *priv             = NULL;
+        DIR                   *dir              = NULL;
+        struct dirent         *entry            = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        snprintf (path, PATH_MAX, "%s/snaps", priv->workdir);
+
+        dir = opendir (path);
+
+        if (!dir) {
+                /* If snaps dir doesn't exists ignore the error for
+                   backward compatibility */
+                if (errno != ENOENT) {
+                        ret = -1;
+                        gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
+                }
+                goto out;
+        }
+
+        glusterd_for_each_entry (entry, dir);
+
+        while (entry) {
+                if (entry->d_type == DT_DIR) {
+                        ret = glusterd_store_retrieve_snap (entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to restore snapshot: %s",
+                                        entry->d_name);
+                                goto out;
+                        }
+                }
+
+                glusterd_for_each_entry (entry, dir);
+        }
+
+        /* Retrieve missed_snaps_list */
+        ret = glusterd_store_retrieve_missed_snaps_list (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Failed to retrieve missed_snaps_list");
+                goto out;
+        }
+
+out:
+        if (dir)
+                closedir (dir);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+/* Writes all the contents of conf->missed_snap_list */
+int32_t
+glusterd_store_write_missed_snapinfo (int32_t fd)
+{
+        char                           value[PATH_MAX]             = "";
+        int32_t                        ret                         = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
+        glusterd_snap_op_t            *snap_opinfo                 = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        /* Write the missed_snap_entry */
+        list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
+                             missed_snaps) {
+                list_for_each_entry (snap_opinfo,
+                                     &missed_snapinfo->snap_ops,
+                                     snap_ops_list) {
+                        snprintf (value, sizeof(value), "%d:%s:%d:%d",
+                                  snap_opinfo->brick_num,
+                                  snap_opinfo->brick_path,
+                                  snap_opinfo->op, snap_opinfo->status);
+                        ret = gf_store_save_value
+                                           (fd,
+                                            missed_snapinfo->node_snap_info,
+                                            value);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to write missed snapinfo");
+                                goto out;
+                        }
+                }
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+/* Adds the missed snap entries to the in-memory conf->missed_snap_list *
+ * and writes them to disk */
+int32_t
+glusterd_store_update_missed_snaps (dict_t *dict, int32_t missed_snap_count)
+{
+        int32_t                        fd                          = -1;
+        int32_t                        ret                         = -1;
+        glusterd_conf_t               *priv                        = NULL;
+        xlator_t                      *this                        = NULL;
+
+        this = THIS;
+        GF_ASSERT(this);
+        GF_ASSERT(dict);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (missed_snap_count < 1) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_store_create_missed_snaps_list_shandle_on_absence ();
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to obtain "
+                        "missed_snaps_list store handle.");
+                goto out;
+        }
+
+        fd = gf_store_mkstemp (priv->missed_snaps_list_shandle);
+        if (fd <= 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to create tmp file");
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_add_missed_snaps_to_list (dict, missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to add missed snaps to list");
+                goto out;
+        }
+
+        ret = glusterd_store_write_missed_snapinfo (fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to write missed snaps to disk");
+                goto out;
+        }
+
+        ret = gf_store_rename_tmppath (priv->missed_snaps_list_shandle);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to rename the tmp file");
+                goto out;
+        }
+out:
+        if (ret && (fd > 0)) {
+                ret = gf_store_unlink_tmppath (priv->missed_snaps_list_shandle);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to unlink the tmp file");
+                }
+                ret = -1;
+        }
+
+        if (fd > 0)
+                close (fd);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo)
+{
+        int32_t                         ret = -1;
+        glusterd_conf_t                 *priv = NULL;
+        char                            peerdir[PATH_MAX] = {0,};
+        char                            filepath[PATH_MAX] = {0,};
+        char                            hostname_path[PATH_MAX] = {0,};
+
+
+        if (!peerinfo) {
+                ret = 0;
+                goto out;
+        }
+
+        priv = THIS->private;
+
+        snprintf (peerdir, PATH_MAX, "%s/peers", priv->workdir);
+
+
+        if (uuid_is_null (peerinfo->uuid)) {
+
+                if (peerinfo->hostname) {
+                        snprintf (filepath, PATH_MAX, "%s/%s", peerdir,
+                                  peerinfo->hostname);
+                } else {
+                       ret = 0;
+                       goto out;
+                }
+        } else {
+
+                snprintf (filepath, PATH_MAX, "%s/%s", peerdir,
+                          uuid_utoa (peerinfo->uuid));
+                snprintf (hostname_path, PATH_MAX, "%s/%s",
+                          peerdir, peerinfo->hostname);
+
+                ret = unlink (hostname_path);
+
+                if (!ret)
+                        goto out;
+        }
+
+        ret = unlink (filepath);
+        if (ret && (errno == ENOENT))
+                ret = 0;
+
+out:
+        if (peerinfo->shandle) {
+                gf_store_handle_destroy (peerinfo->shandle);
+                peerinfo->shandle = NULL;
+        }
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+void
+glusterd_store_peerinfo_dirpath_set (char *path, size_t len)
+{
+        glusterd_conf_t                 *priv = NULL;
+        GF_ASSERT (path);
+        GF_ASSERT (len >= PATH_MAX);
+
+        priv = THIS->private;
+        snprintf (path, len, "%s/peers", priv->workdir);
+}
+
+int32_t
+glusterd_store_create_peer_dir ()
+{
+        int32_t                             ret = 0;
+        char                            path[PATH_MAX];
+
+        glusterd_store_peerinfo_dirpath_set (path, sizeof (path));
+        ret = gf_store_mkdir (path);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+static void
+glusterd_store_uuid_peerpath_set (glusterd_peerinfo_t *peerinfo, char *peerfpath,
+                             size_t len)
+{
+        char                    peerdir[PATH_MAX];
+        char                    str[50] = {0};
+
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (peerfpath);
+        GF_ASSERT (len >= PATH_MAX);
+
+        glusterd_store_peerinfo_dirpath_set (peerdir, sizeof (peerdir));
+        uuid_unparse (peerinfo->uuid, str);
+        snprintf (peerfpath, len, "%s/%s", peerdir, str);
+}
+
+static void
+glusterd_store_hostname_peerpath_set (glusterd_peerinfo_t *peerinfo,
+                                       char *peerfpath, size_t len)
+{
+        char                    peerdir[PATH_MAX];
+
+        GF_ASSERT (peerinfo);
+        GF_ASSERT (peerfpath);
+        GF_ASSERT (len >= PATH_MAX);
+
+        glusterd_store_peerinfo_dirpath_set (peerdir, sizeof (peerdir));
+        snprintf (peerfpath, len, "%s/%s", peerdir, peerinfo->hostname);
+}
+
+int32_t
+glusterd_store_peerinfo_hostname_shandle_create (glusterd_peerinfo_t *peerinfo)
+{
+        char                    peerfpath[PATH_MAX];
+        int32_t                 ret = -1;
+
+        glusterd_store_hostname_peerpath_set (peerinfo, peerfpath,
+                                              sizeof (peerfpath));
+        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
+                                                 peerfpath);
+        return ret;
+}
+
+int32_t
+glusterd_store_peerinfo_uuid_shandle_create (glusterd_peerinfo_t *peerinfo)
+{
+        char                    peerfpath[PATH_MAX];
+        int32_t                 ret = -1;
+
+        glusterd_store_uuid_peerpath_set (peerinfo, peerfpath,
+                                          sizeof (peerfpath));
+        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
+                                                 peerfpath);
+        return ret;
+}
+
+int32_t
+glusterd_peerinfo_hostname_shandle_check_destroy (glusterd_peerinfo_t *peerinfo)
+{
+        char                    peerfpath[PATH_MAX];
+        int32_t                 ret = -1;
+        struct  stat            stbuf = {0,};
+
+        glusterd_store_hostname_peerpath_set (peerinfo, peerfpath,
+                                              sizeof (peerfpath));
+        ret = stat (peerfpath, &stbuf);
+        if (!ret) {
+                if (peerinfo->shandle)
+                        gf_store_handle_destroy (peerinfo->shandle);
+                peerinfo->shandle = NULL;
+                ret = unlink (peerfpath);
+        }
+        return ret;
+}
+
+int32_t
+glusterd_store_create_peer_shandle (glusterd_peerinfo_t *peerinfo)
+{
+        int32_t                 ret = 0;
+
+        GF_ASSERT (peerinfo);
+
+        if (glusterd_peerinfo_is_uuid_unknown (peerinfo)) {
+                ret = glusterd_store_peerinfo_hostname_shandle_create (peerinfo);
+        } else {
+                ret = glusterd_peerinfo_hostname_shandle_check_destroy (peerinfo);
+                ret = glusterd_store_peerinfo_uuid_shandle_create (peerinfo);
+        }
+        return ret;
+}
+
+int32_t
+glusterd_store_peer_write (int fd, glusterd_peerinfo_t *peerinfo)
+{
+        char                    buf[50] = {0};
+        int32_t                 ret = 0;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_UUID,
+                                   uuid_utoa (peerinfo->uuid));
+        if (ret)
+                goto out;
+
+        snprintf (buf, sizeof (buf), "%d", peerinfo->state.state);
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_STATE, buf);
+        if (ret)
+                goto out;
+
+        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_HOSTNAME "1",
+                                   peerinfo->hostname);
+        if (ret)
+                goto out;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_perform_peer_store (glusterd_peerinfo_t *peerinfo)
+{
+        int                                 fd = -1;
+        int32_t                             ret = -1;
+
+        GF_ASSERT (peerinfo);
+
+        fd = gf_store_mkstemp (peerinfo->shandle);
+        if (fd <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_store_peer_write (fd, peerinfo);
+        if (ret)
+                goto out;
+
+        ret = gf_store_rename_tmppath (peerinfo->shandle);
+out:
+        if (ret && (fd > 0))
+                gf_store_unlink_tmppath (peerinfo->shandle);
+        if (fd > 0)
+                close (fd);
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_peerinfo (glusterd_peerinfo_t *peerinfo)
+{
+        int32_t                         ret = -1;
+
+        GF_ASSERT (peerinfo);
+
+        ret = glusterd_store_create_peer_dir ();
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_create_peer_shandle (peerinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_perform_peer_store (peerinfo);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_store_retrieve_peers (xlator_t *this)
+{
+        int32_t                 ret = 0;
+        glusterd_conf_t         *priv = NULL;
+        DIR                     *dir = NULL;
+        struct dirent           *entry = NULL;
+        char                    path[PATH_MAX] = {0,};
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        uuid_t                  uuid = {0,};
+        char                    *hostname = NULL;
+        int32_t                 state = 0;
+        gf_store_handle_t       *shandle = NULL;
+        char                    filepath[PATH_MAX] = {0,};
+        gf_store_iter_t         *iter = NULL;
+        char                    *key = NULL;
+        char                    *value = NULL;
+        glusterd_peerctx_args_t args = {0};
+        gf_store_op_errno_t     op_errno = GD_STORE_SUCCESS;
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
+                  GLUSTERD_PEER_DIR_PREFIX);
+
+        dir = opendir (path);
+
+        if (!dir) {
+                gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
+                ret = -1;
+                goto out;
+        }
+
+        glusterd_for_each_entry (entry, dir);
+
+        while (entry) {
+                snprintf (filepath, PATH_MAX, "%s/%s", path, entry->d_name);
+                ret = gf_store_handle_retrieve (filepath, &shandle);
+                if (ret)
+                        goto out;
+
+                ret = gf_store_iter_new (shandle, &iter);
+                if (ret)
+                        goto out;
+
+                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+                if (ret)
+                        goto out;
+
+                while (!ret) {
+
+                        if (!strncmp (GLUSTERD_STORE_KEY_PEER_UUID, key,
+                                      strlen (GLUSTERD_STORE_KEY_PEER_UUID))) {
+                                if (value)
+                                        uuid_parse (value, uuid);
+                        } else if (!strncmp (GLUSTERD_STORE_KEY_PEER_STATE,
+                                    key,
+                                    strlen (GLUSTERD_STORE_KEY_PEER_STATE))) {
+                                state = atoi (value);
+                        } else if (!strncmp (GLUSTERD_STORE_KEY_PEER_HOSTNAME,
+                                   key,
+                                   strlen (GLUSTERD_STORE_KEY_PEER_HOSTNAME))) {
+                                hostname = gf_strdup (value);
+                        } else {
+                                gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
+                                        key);
+                        }
+
+                        GF_FREE (key);
+                        GF_FREE (value);
+                        key = NULL;
+                        value = NULL;
+
+                        ret = gf_store_iter_get_next (iter, &key, &value,
+                                                      &op_errno);
+                }
+                if (op_errno != GD_STORE_EOF)
+                        goto out;
+
+                (void) gf_store_iter_destroy (iter);
+
+                ret = glusterd_friend_add (hostname, 0, state, &uuid,
+                                           &peerinfo, 1, NULL);
+
+                GF_FREE (hostname);
+                if (ret)
+                        goto out;
+
+                peerinfo->shandle = shandle;
+                glusterd_for_each_entry (entry, dir);
+        }
+
+        args.mode = GD_MODE_ON;
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+                ret = glusterd_friend_rpc_create (this, peerinfo, &args);
+                if (ret)
+                        goto out;
+        }
+
+out:
+        if (dir)
+                closedir (dir);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_resolve_all_bricks (xlator_t  *this)
+{
+        int32_t                 ret = 0;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+
+        GF_ASSERT (this);
+        priv = this->private;
+
+        GF_ASSERT (priv);
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        ret = glusterd_resolve_brick (brickinfo);
+                        if (ret) {
+                                gf_log ("glusterd", GF_LOG_ERROR,
+                                        "resolve brick failed in restore");
+                                goto out;
+                        }
+                }
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_restore ()
+{
+        int32_t         ret = -1;
+        xlator_t        *this = NULL;
+
+        this = THIS;
+
+        ret = glusterd_restore_op_version (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to restore op_version");
+                goto out;
+        }
+
+        ret = glusterd_store_retrieve_volumes (this, NULL);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_retrieve_snaps (this);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_retrieve_peers (this);
+        if (ret)
+                goto out;
+
+        ret = glusterd_resolve_all_bricks (this);
+        if (ret)
+                goto out;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
new file mode 100644
index 000000000..1b5cebc0c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -0,0 +1,164 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_HA_H_
+#define _GLUSTERD_HA_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <pthread.h>
+#include "uuid.h"
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "run.h"
+#include "logging.h"
+#include "call-stub.h"
+#include "fd.h"
+#include "byte-order.h"
+#include "glusterd.h"
+#include "rpcsvc.h"
+
+typedef enum glusterd_store_ver_ac_{
+        GLUSTERD_VOLINFO_VER_AC_NONE = 0,
+        GLUSTERD_VOLINFO_VER_AC_INCREMENT = 1,
+        GLUSTERD_VOLINFO_VER_AC_DECREMENT = 2,
+} glusterd_volinfo_ver_ac_t;
+
+
+#define GLUSTERD_STORE_UUID_KEY                 "UUID"
+
+#define GLUSTERD_STORE_KEY_VOL_TYPE             "type"
+#define GLUSTERD_STORE_KEY_VOL_COUNT            "count"
+#define GLUSTERD_STORE_KEY_VOL_STATUS           "status"
+#define GLUSTERD_STORE_KEY_VOL_PORT             "port"
+#define GLUSTERD_STORE_KEY_VOL_SUB_COUNT        "sub_count"
+#define GLUSTERD_STORE_KEY_VOL_STRIPE_CNT       "stripe_count"
+#define GLUSTERD_STORE_KEY_VOL_REPLICA_CNT      "replica_count"
+#define GLUSTERD_STORE_KEY_VOL_BRICK            "brick"
+#define GLUSTERD_STORE_KEY_VOL_VERSION          "version"
+#define GLUSTERD_STORE_KEY_VOL_TRANSPORT        "transport-type"
+#define GLUSTERD_STORE_KEY_VOL_ID               "volume-id"
+#define GLUSTERD_STORE_KEY_VOL_IS_RESTORED      "is-volume-restored"
+#define GLUSTERD_STORE_KEY_RB_STATUS            "rb_status"
+#define GLUSTERD_STORE_KEY_RB_SRC_BRICK         "rb_src"
+#define GLUSTERD_STORE_KEY_RB_DST_BRICK         "rb_dst"
+#define GLUSTERD_STORE_KEY_RB_DST_PORT          "rb_port"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG           "rebalance_status"
+#define GLUSTERD_STORE_KEY_DEFRAG_OP            "rebalance_op"
+#define GLUSTERD_STORE_KEY_USERNAME             "username"
+#define GLUSTERD_STORE_KEY_PASSWORD             "password"
+#define GLUSTERD_STORE_KEY_PARENT_VOLNAME       "parent_volname"
+#define GLUSTERD_STORE_KEY_VOL_OP_VERSION       "op-version"
+#define GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION "client-op-version"
+
+#define GLUSTERD_STORE_KEY_SNAP_NAME            "name"
+#define GLUSTERD_STORE_KEY_SNAP_ID              "snap-id"
+#define GLUSTERD_STORE_KEY_SNAP_DESC            "desc"
+#define GLUSTERD_STORE_KEY_SNAP_TIMESTAMP       "time-stamp"
+#define GLUSTERD_STORE_KEY_SNAP_STATUS          "status"
+#define GLUSTERD_STORE_KEY_SNAP_RESTORED        "snap-restored"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT  "snap-max-hard-limit"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT  "snap-max-soft-limit"
+
+#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME       "hostname"
+#define GLUSTERD_STORE_KEY_BRICK_PATH           "path"
+#define GLUSTERD_STORE_KEY_BRICK_PORT           "listen-port"
+#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT      "rdma.listen-port"
+#define GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED "decommissioned"
+#define GLUSTERD_STORE_KEY_BRICK_VGNAME         "vg"
+#define GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH    "device_path"
+#define GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS    "snap-status"
+
+#define GLUSTERD_STORE_KEY_PEER_UUID            "uuid"
+#define GLUSTERD_STORE_KEY_PEER_HOSTNAME        "hostname"
+#define GLUSTERD_STORE_KEY_PEER_STATE           "state"
+
+#define GLUSTERD_STORE_KEY_VOL_CAPS             "caps"
+
+#define glusterd_for_each_entry(entry, dir) \
+        do {\
+                entry = NULL;\
+                if (dir) {\
+                        entry = readdir (dir);\
+                        while (entry && (!strcmp (entry->d_name, ".") ||\
+                            !fnmatch ("*.tmp", entry->d_name, 0) ||\
+                            !strcmp (entry->d_name, ".."))) {\
+                                entry = readdir (dir);\
+                        }\
+                }\
+        } while (0); \
+
+
+int32_t
+glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac);
+
+int32_t
+glusterd_store_delete_volume (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_store_delete_snap (glusterd_snap_t *snap);
+
+int32_t
+glusterd_retrieve_uuid ();
+
+int32_t
+glusterd_store_peerinfo (glusterd_peerinfo_t *peerinfo);
+
+int32_t
+glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo);
+
+int32_t
+glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo,
+                             char *delete_path);
+
+int32_t
+glusterd_restore ();
+
+void
+glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
+                                         glusterd_volinfo_ver_ac_t ac);
+gf_boolean_t
+glusterd_store_is_valid_brickpath (char *volname, char *brick);
+
+int32_t
+glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_retrieve_op_version (xlator_t *this, int *op_version);
+
+int
+glusterd_store_global_info (xlator_t *this);
+
+int32_t
+glusterd_store_retrieve_options (xlator_t *this);
+
+int32_t
+glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_store_options (xlator_t *this, dict_t *opts);
+
+void
+glusterd_replace_slash_with_hyphen (char *str);
+
+int32_t
+glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_store_snap (glusterd_snap_t *snap);
+
+int32_t
+glusterd_store_update_missed_snaps (dict_t *dict,
+                                    int32_t missed_snap_count);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
new file mode 100644
index 000000000..438df8266
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -0,0 +1,1639 @@
+/*
+   Copyright (c) 2012-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+/* rpc related syncops */
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+#include "xdr-generic.h"
+#include "glusterd1-xdr.h"
+#include "glusterd-syncop.h"
+
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+
+extern glusterd_op_info_t opinfo;
+
+void
+gd_synctask_barrier_wait (struct syncargs *args, int count)
+{
+        glusterd_conf_t *conf = THIS->private;
+
+        synclock_unlock (&conf->big_lock);
+        synctask_barrier_wait (args, count);
+        synclock_lock (&conf->big_lock);
+
+	syncbarrier_destroy (&args->barrier);
+}
+
+static void
+gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                           char *op_errstr, int op_code,
+                           glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char     err_str[PATH_MAX] = "Please check log file for details.";
+        char     op_err[PATH_MAX] = "";
+        int      len               = -1;
+        char    *peer_str      = NULL;
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, "")) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "Error: %s", op_errstr);
+                        err_str[len] = '\0';
+                }
+
+                switch (op_code){
+                        case GLUSTERD_MGMT_V3_LOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Locking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_V3_UNLOCK:
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Unlocking failed "
+                                                "on %s. %s", peer_str, err_str);
+                                break;
+                        }
+                }
+                op_err[len] = '\0';
+
+                if (args->errstr) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s\n%s", args->errstr,
+                                        op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s", op_err);
+                err_str[len] = '\0';
+
+                gf_log ("", GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+static void
+gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
+                   char *op_errstr, int op_code,
+                   glusterd_peerinfo_t *peerinfo, u_char *uuid)
+{
+        char     err_str[PATH_MAX] = "Please check log file for details.";
+        char     op_err[PATH_MAX] = "";
+        int      len               = -1;
+        char    *peer_str      = NULL;
+
+        if (op_ret) {
+                args->op_ret = op_ret;
+                args->op_errno = op_errno;
+
+                if (peerinfo)
+                        peer_str = peerinfo->hostname;
+                else
+                        peer_str = uuid_utoa (uuid);
+
+                if (op_errstr && strcmp (op_errstr, "")) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "Error: %s", op_errstr);
+                        err_str[len] = '\0';
+                }
+
+                switch (op_code){
+                        case GLUSTERD_MGMT_CLUSTER_UNLOCK :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Unlocking failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_STAGE_OP :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Staging failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                        case GLUSTERD_MGMT_COMMIT_OP :
+                        {
+                                len = snprintf (op_err, sizeof(op_err) - 1,
+                                                "Commit failed on %s. %s",
+                                                peer_str, err_str);
+                                break;
+                        }
+                }
+                op_err[len] = '\0';
+
+                if (args->errstr) {
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s\n%s", args->errstr,
+                                        op_err);
+                        GF_FREE (args->errstr);
+                        args->errstr = NULL;
+                } else
+                        len = snprintf (err_str, sizeof(err_str) - 1,
+                                        "%s", op_err);
+                err_str[len] = '\0';
+
+                gf_log ("", GF_LOG_ERROR, "%s", op_err);
+                args->errstr = gf_strdup (err_str);
+        }
+
+        return;
+}
+
+void
+gd_syncargs_init (struct syncargs *args, dict_t *op_ctx)
+{
+        args->dict = op_ctx;
+        pthread_mutex_init (&args->lock_dict, NULL);
+}
+
+static void
+gd_stage_op_req_free (gd1_mgmt_stage_op_req *req)
+{
+        if (!req)
+                return;
+
+        GF_FREE (req->buf.buf_val);
+        GF_FREE (req);
+}
+
+static void
+gd_commit_op_req_free (gd1_mgmt_commit_op_req *req)
+{
+        if (!req)
+                return;
+
+        GF_FREE (req->buf.buf_val);
+        GF_FREE (req);
+}
+
+static void
+gd_brick_op_req_free (gd1_mgmt_brick_op_req *req)
+{
+        if (!req)
+                return;
+
+        if (strcmp (req->name, "") != 0)
+                GF_FREE (req->name);
+        GF_FREE (req->input.input_val);
+        GF_FREE (req);
+}
+
+int
+gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
+                          void *cookie, rpc_clnt_prog_t *prog, int procnum,
+                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+        int            ret      = -1;
+        struct iobuf  *iobuf    = NULL;
+        struct iobref *iobref   = NULL;
+        int            count    = 0;
+        struct iovec   iov      = {0, };
+        ssize_t        req_size = 0;
+        call_frame_t  *frame    = NULL;
+
+        GF_ASSERT (rpc);
+        if (!req)
+                goto out;
+
+        req_size = xdr_sizeof (xdrproc, req);
+        iobuf = iobuf_get2 (rpc->ctx->iobuf_pool, req_size);
+        if (!iobuf)
+                goto out;
+
+        iobref = iobref_new ();
+        if (!iobref)
+                goto out;
+
+        frame = create_frame (THIS, THIS->ctx->pool);
+        if (!frame)
+                goto out;
+
+        iobref_add (iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len  = iobuf_pagesize (iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic (iov, req, xdrproc);
+        if (ret == -1)
+                goto out;
+
+        iov.iov_len = ret;
+        count = 1;
+
+        frame->local = local;
+        frame->cookie = cookie;
+
+        /* Send the msg */
+        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn,
+                               &iov, count, NULL, 0, iobref,
+                               frame, NULL, 0, NULL, 0, NULL);
+
+        /* TODO: do we need to start ping also? */
+
+out:
+        iobref_unref (iobref);
+        iobuf_unref (iobuf);
+
+        return ret;
+}
+
+/* Defined in glusterd-rpc-ops.c */
+extern struct rpc_clnt_program gd_mgmt_prog;
+extern struct rpc_clnt_program gd_brick_prog;
+extern struct rpc_clnt_program gd_mgmt_v3_prog;
+
+int
+glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
+{
+        int ret = 0;
+
+        switch (op) {
+        case GD_OP_REPLACE_BRICK:
+                ret = glusterd_rb_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_SYNC_VOLUME:
+                ret = glusterd_sync_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_profile_volume_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_GSYNC_CREATE:
+        break;
+
+        case GD_OP_GSYNC_SET:
+                ret = glusterd_gsync_use_rsp_dict (aggr, rsp, NULL);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_STATUS_VOLUME:
+                ret = glusterd_volume_status_copy_to_op_ctx_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = glusterd_volume_rebalance_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_HEAL_VOLUME:
+                ret = glusterd_volume_heal_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+
+        break;
+
+        case GD_OP_QUOTA:
+        case GD_OP_CLEARLOCKS_VOLUME:
+                ret = glusterd_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+
+        break;
+
+        case GD_OP_SYS_EXEC:
+                ret = glusterd_sys_exec_output_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        case GD_OP_SNAP:
+                ret = glusterd_snap_use_rsp_dict (aggr, rsp);
+                if (ret)
+                        goto out;
+        break;
+
+        default:
+        break;
+        }
+out:
+        return ret;
+}
+
+int32_t
+_gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_cluster_lock_rsp   rsp         = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        /* Set peer as locked, so we unlock only the locked peers */
+        if (rsp.op_ret == 0)
+                peerinfo->locked = _gf_true;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_collate_errors (args, op_ret, op_errno, NULL,
+                           GLUSTERD_MGMT_CLUSTER_LOCK, peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_lock_cbk);
+}
+
+int
+gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                     uuid_t my_uuid, uuid_t recv_uuid)
+{
+        int                       ret = -1;
+        gd1_mgmt_cluster_lock_req req  = {{0},};
+        glusterd_conf_t           *conf = THIS->private;
+
+        uuid_copy (req.uuid, my_uuid);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_CLUSTER_LOCK,
+                                        gd_syncop_mgmt_lock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+        synclock_lock (&conf->big_lock);
+        return ret;
+}
+
+int32_t
+gd_syncop_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                               int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_v3_lock_rsp         rsp        = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        GF_ASSERT(req);
+        GF_ASSERT(iov);
+        GF_ASSERT(myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_LOCK,
+                                   peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_syncop_mgmt_v3_lock_cbk_fn);
+}
+
+int
+gd_syncop_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
+                        glusterd_peerinfo_t *peerinfo,
+                        struct syncargs *args, uuid_t my_uuid,
+                        uuid_t recv_uuid, uuid_t txn_id)
+{
+        int                      ret  = -1;
+        gd1_mgmt_v3_lock_req     req  = {{0},};
+        glusterd_conf_t         *conf = THIS->private;
+
+        GF_ASSERT(op_ctx);
+        GF_ASSERT(peerinfo);
+        GF_ASSERT(args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        uuid_copy (req.txn_id, txn_id);
+        req.op = op;
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_LOCK,
+                                        gd_syncop_mgmt_v3_lock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_lock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+gd_syncop_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_v3_unlock_rsp       rsp        = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        GF_ASSERT(req);
+        GF_ASSERT(iov);
+        GF_ASSERT(myframe);
+
+        frame  = myframe;
+        args   = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+        frame->cookie = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        /* Set peer as locked, so we unlock only the locked peers */
+        if (rsp.op_ret == 0)
+                peerinfo->locked = _gf_true;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
+                                   GLUSTERD_MGMT_V3_UNLOCK,
+                                   peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                              int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        gd_syncop_mgmt_v3_unlock_cbk_fn);
+}
+
+int
+gd_syncop_mgmt_v3_unlock (dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
+                          struct syncargs *args, uuid_t my_uuid,
+                          uuid_t recv_uuid, uuid_t txn_id)
+{
+        int                          ret  = -1;
+        gd1_mgmt_v3_unlock_req       req  = {{0},};
+        glusterd_conf_t             *conf = THIS->private;
+
+        GF_ASSERT(op_ctx);
+        GF_ASSERT(peerinfo);
+        GF_ASSERT(args);
+
+        ret = dict_allocate_and_serialize (op_ctx,
+                                           &req.dict.dict_val,
+                                           &req.dict.dict_len);
+        if (ret)
+                goto out;
+
+        uuid_copy (req.uuid, my_uuid);
+        uuid_copy (req.txn_id, txn_id);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_v3_prog,
+                                        GLUSTERD_MGMT_V3_UNLOCK,
+                                        gd_syncop_mgmt_v3_unlock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_v3_unlock_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+_gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                            int count, void *myframe)
+{
+        int                         ret         = -1;
+        struct syncargs             *args       = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        gd1_mgmt_cluster_unlock_rsp rsp         = {{0},};
+        call_frame_t                *frame      = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        frame = myframe;
+        args  = frame->local;
+        peerinfo = frame->cookie;
+        frame->local = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+        if (ret < 0)
+                goto out;
+
+        uuid_copy (args->uuid, rsp.uuid);
+
+        peerinfo->locked = _gf_false;
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+out:
+        gd_collate_errors (args, op_ret, op_errno, NULL,
+                           GLUSTERD_MGMT_CLUSTER_UNLOCK, peerinfo, rsp.uuid);
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
+                           int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_mgmt_unlock_cbk);
+}
+
+
+int
+gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                       uuid_t my_uuid, uuid_t recv_uuid)
+{
+        int                         ret     = -1;
+        gd1_mgmt_cluster_unlock_req req     = {{0},};
+        glusterd_conf_t             *conf   = THIS->private;
+
+        uuid_copy (req.uuid, my_uuid);
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
+                                        &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_CLUSTER_UNLOCK,
+                                        gd_syncop_mgmt_unlock_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+        synclock_lock (&conf->big_lock);
+        return ret;
+}
+
+int32_t
+_gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        int                         ret         = -1;
+        gd1_mgmt_stage_op_rsp       rsp         = {{0},};
+        struct syncargs             *args       = NULL;
+        xlator_t                    *this       = NULL;
+        dict_t                      *rsp_dict   = NULL;
+        call_frame_t                *frame      = NULL;
+        glusterd_peerinfo_t         *peerinfo   = NULL;
+        int                         op_ret      = -1;
+        int                         op_errno    = -1;
+
+        this  = THIS;
+        frame = myframe;
+        args  = frame->local;
+        frame->local = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Staging response "
+                        "for 'Volume %s' received from unknown "
+                        "peer: %s", gd_op_list[rsp.op],
+                        uuid_utoa (rsp.uuid));
+                goto out;
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        if (rsp.op == GD_OP_REPLACE_BRICK) {
+                pthread_mutex_lock (&args->lock_dict);
+                {
+                        ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
+                                                             rsp_dict);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR, "%s",
+                                        "Failed to aggregate response from "
+                                        " node/brick");
+                }
+                pthread_mutex_unlock (&args->lock_dict);
+        }
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
+                           GLUSTERD_MGMT_STAGE_OP, peerinfo, rsp.uuid);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+        return 0;
+}
+
+int32_t
+gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_stage_op_cbk);
+}
+
+
+int
+gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *args,
+                         uuid_t my_uuid, uuid_t recv_uuid, int op,
+                         dict_t *dict_out, dict_t *op_ctx)
+{
+        gd1_mgmt_stage_op_req *req  = NULL;
+        glusterd_conf_t       *conf = THIS->private;
+        int                   ret   = -1;
+
+        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_stage_req_t);
+        if (!req)
+                goto out;
+
+        uuid_copy (req->uuid, my_uuid);
+        req->op = op;
+
+        ret = dict_allocate_and_serialize (dict_out,
+                                           &req->buf.buf_val, &req->buf.buf_len);
+        if (ret)
+                goto out;
+
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (rpc, req, args, NULL, &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_STAGE_OP,
+                                        gd_syncop_stage_op_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_stage_op_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gd_stage_op_req_free (req);
+        return ret;
+
+}
+
+int32_t
+_gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        struct syncargs        *args  = NULL;
+        gd1_mgmt_brick_op_rsp  rsp   = {0,};
+        int                    ret   = -1;
+        call_frame_t           *frame = NULL;
+
+        frame = myframe;
+        args = frame->local;
+        frame->local = NULL;
+
+        /* initialize */
+        args->op_ret   = -1;
+        args->op_errno = EINVAL;
+
+        if (-1 == req->rpc_status) {
+                args->op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+        if (ret < 0)
+                goto out;
+
+        if (rsp.output.output_len) {
+                args->dict  = dict_new ();
+                if (!args->dict) {
+                        ret = -1;
+                        args->op_errno = ENOMEM;
+                        goto out;
+                }
+
+                ret = dict_unserialize (rsp.output.output_val,
+                                        rsp.output.output_len,
+                                        &args->dict);
+                if (ret < 0)
+                        goto out;
+        }
+
+        args->op_ret = rsp.op_ret;
+        args->op_errno = rsp.op_errno;
+        args->errstr = gf_strdup (rsp.op_errstr);
+
+out:
+        if ((rsp.op_errstr) && (strcmp (rsp.op_errstr, "") != 0))
+                free (rsp.op_errstr);
+        free (rsp.output.output_val);
+
+        STACK_DESTROY (frame->root);
+        __wake (args);
+
+        return 0;
+}
+
+int32_t
+gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
+                        int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_brick_op_cbk);
+}
+
+int
+gd_syncop_mgmt_brick_op (struct rpc_clnt *rpc, glusterd_pending_node_t *pnode,
+                         int op, dict_t *dict_out, dict_t *op_ctx,
+                         char **errstr)
+{
+        struct syncargs        args = {0, };
+        gd1_mgmt_brick_op_req  *req  = NULL;
+        int                    ret  = 0;
+        xlator_t               *this = NULL;
+
+        this = THIS;
+        args.op_ret = -1;
+        args.op_errno = ENOTCONN;
+
+        if ((pnode->type == GD_NODE_NFS) ||
+            ((pnode->type == GD_NODE_SHD) &&
+            (op == GD_OP_STATUS_VOLUME))) {
+                ret = glusterd_node_op_build_payload
+                        (op, &req, dict_out);
+
+        } else {
+                ret = glusterd_brick_op_build_payload
+                        (op, pnode->node, &req, dict_out);
+
+        }
+
+        if (ret)
+                goto out;
+
+        GD_SYNCOP (rpc, (&args), NULL, gd_syncop_brick_op_cbk, req,
+                   &gd_brick_prog, req->op, xdr_gd1_mgmt_brick_op_req);
+
+        if (args.errstr) {
+                if ((strlen(args.errstr) > 0) && errstr)
+                        *errstr = args.errstr;
+                else
+                        GF_FREE (args.errstr);
+        }
+
+        if (GD_OP_STATUS_VOLUME == op) {
+                ret = dict_set_int32 (args.dict, "index", pnode->index);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting index on brick status"
+                                " rsp dict");
+                        args.op_ret = -1;
+                        goto out;
+                }
+        }
+        if (args.op_ret == 0)
+                glusterd_handle_node_rsp (dict_out, pnode->node, op,
+                                          args.dict, op_ctx, errstr,
+                                          pnode->type);
+
+out:
+        errno = args.op_errno;
+        if (args.dict)
+                dict_unref (args.dict);
+        gd_brick_op_req_free (req);
+        return args.op_ret;
+
+}
+
+int32_t
+_gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                          int count, void *myframe)
+{
+        int                          ret           = -1;
+        gd1_mgmt_commit_op_rsp       rsp           = {{0},};
+        struct syncargs             *args          = NULL;
+        xlator_t                    *this          = NULL;
+        dict_t                      *rsp_dict      = NULL;
+        call_frame_t                *frame         = NULL;
+        glusterd_peerinfo_t         *peerinfo      = NULL;
+        int                          op_ret        = -1;
+        int                          op_errno      = -1;
+
+        this  = THIS;
+        frame = myframe;
+        args  = frame->local;
+        frame->local = NULL;
+
+        if (-1 == req->rpc_status) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        ret = xdr_to_generic (*iov, &rsp,
+                              (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+        if (ret < 0) {
+                goto out;
+        }
+
+        if (rsp.dict.dict_len) {
+                /* Unserialize the dictionary */
+                rsp_dict  = dict_new ();
+
+                ret = dict_unserialize (rsp.dict.dict_val,
+                                        rsp.dict.dict_len,
+                                        &rsp_dict);
+                if (ret < 0) {
+                        GF_FREE (rsp.dict.dict_val);
+                        goto out;
+                } else {
+                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
+                }
+        }
+
+        ret = glusterd_friend_find (rsp.uuid, NULL, &peerinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Commit response "
+                        "for 'Volume %s' received from unknown "
+                        "peer: %s", gd_op_list[rsp.op],
+                        uuid_utoa (rsp.uuid));
+                goto out;
+        }
+
+        uuid_copy (args->uuid, rsp.uuid);
+        pthread_mutex_lock (&args->lock_dict);
+        {
+                ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
+                                                     rsp_dict);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                "Failed to aggregate response from "
+                                " node/brick");
+        }
+        pthread_mutex_unlock (&args->lock_dict);
+
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+
+out:
+        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
+                           GLUSTERD_MGMT_COMMIT_OP, peerinfo, rsp.uuid);
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        STACK_DESTROY (frame->root);
+        synctask_barrier_wake(args);
+
+        return 0;
+}
+
+int32_t
+gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe)
+{
+        return glusterd_big_locked_cbk (req, iov, count, myframe,
+                                        _gd_syncop_commit_op_cbk);
+}
+
+
+int
+gd_syncop_mgmt_commit_op (struct rpc_clnt *rpc, struct syncargs *args,
+                          uuid_t my_uuid, uuid_t recv_uuid,
+                          int op, dict_t *dict_out, dict_t *op_ctx)
+{
+        glusterd_conf_t        *conf = THIS->private;
+        gd1_mgmt_commit_op_req *req  = NULL;
+        int                    ret  = -1;
+
+        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_commit_req_t);
+        if (!req)
+                goto out;
+
+        uuid_copy (req->uuid, my_uuid);
+        req->op = op;
+
+        ret = dict_allocate_and_serialize (dict_out,
+                                           &req->buf.buf_val, &req->buf.buf_len);
+        if (ret)
+                goto out;
+
+        synclock_unlock (&conf->big_lock);
+        ret = gd_syncop_submit_request (rpc, req, args, NULL, &gd_mgmt_prog,
+                                        GLUSTERD_MGMT_COMMIT_OP ,
+                                        gd_syncop_commit_op_cbk,
+                                        (xdrproc_t) xdr_gd1_mgmt_commit_op_req);
+        synclock_lock (&conf->big_lock);
+out:
+        gd_commit_op_req_free (req);
+        return ret;
+}
+
+
+int
+gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers,
+                     glusterd_op_t op)
+{
+        glusterd_peerinfo_t *peerinfo = NULL;
+        int                 npeers      = 0;
+
+        list_for_each_entry (peerinfo, peers, uuid_list) {
+                if (!peerinfo->connected)
+                        continue;
+                if (op != GD_OP_SYNC_VOLUME &&
+                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+                        continue;
+
+                list_add_tail (&peerinfo->op_peers_list, xact_peers);
+                npeers++;
+        }
+        return npeers;
+}
+
+int
+gd_lock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, dict_t *op_ctx,
+                  char **op_errstr, int npeers, uuid_t txn_id)
+{
+        int                 ret         = -1;
+        int                 peer_cnt    = 0;
+        uuid_t              peer_uuid   = {0};
+        xlator_t            *this       = NULL;
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        struct syncargs     args        = {0};
+        struct list_head   *peers       = NULL;
+
+        peers = &conf->xaction_peers;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        this = THIS;
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                if (conf->op_version < 3) {
+                        /* Reset lock status */
+                        peerinfo->locked = _gf_false;
+                        gd_syncop_mgmt_lock (peerinfo, &args,
+                                             MY_UUID, peer_uuid);
+                } else
+                        gd_syncop_mgmt_v3_lock (op, op_ctx, peerinfo, &args,
+                                                MY_UUID, peer_uuid, txn_id);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.op_ret) {
+                if (args.errstr)
+                         *op_errstr = gf_strdup (args.errstr);
+                else {
+                        ret = gf_asprintf (op_errstr, "Another transaction could be "
+                                           "in progress. Please try again after "
+                                           "sometime.");
+                        if (ret == -1)
+                                *op_errstr = NULL;
+
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to acquire lock");
+
+                }
+        }
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent lock op req for 'Volume %s' "
+                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+out:
+        return ret;
+}
+
+int
+gd_stage_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
+                   dict_t *req_dict, char **op_errstr, int npeers)
+{
+        int                 ret             = -1;
+        int                 peer_cnt           = 0;
+        dict_t              *rsp_dict       = NULL;
+        char                *hostname       = NULL;
+        xlator_t            *this           = NULL;
+        glusterd_peerinfo_t *peerinfo       = NULL;
+        uuid_t              tmp_uuid        = {0};
+        char                *errstr         = NULL;
+        struct syncargs     args            = {0};
+
+        this = THIS;
+        rsp_dict = dict_new ();
+        if (!rsp_dict)
+                goto out;
+
+        ret = glusterd_op_stage_validate (op, req_dict, op_errstr, rsp_dict);
+        if (ret) {
+                hostname = "localhost";
+                goto stage_done;
+        }
+
+        if ((op == GD_OP_REPLACE_BRICK)) {
+                ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                "Failed to aggregate response from node/brick");
+                        goto out;
+                }
+        }
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+stage_done:
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_STAGE_FAIL,
+                        gd_op_list[op], hostname, (*op_errstr) ? ":" : " ",
+                        (*op_errstr) ? *op_errstr : " ");
+                if (*op_errstr == NULL)
+                        gf_asprintf (op_errstr, OPERRSTR_STAGE_FAIL, hostname);
+                goto out;
+        }
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        gd_syncargs_init (&args, op_ctx);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                ret = gd_syncop_mgmt_stage_op (peerinfo->rpc, &args,
+                                               MY_UUID, tmp_uuid,
+                                               op, req_dict, op_ctx);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+
+        if (args.errstr)
+                 *op_errstr = gf_strdup (args.errstr);
+        else if (dict_get_str (op_ctx, "errstr", &errstr) == 0)
+                *op_errstr = gf_strdup (errstr);
+
+        ret = args.op_ret;
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent stage op req for 'Volume %s' "
+                "to %d peers", gd_op_list[op], peer_cnt);
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+        return ret;
+}
+
+int
+gd_commit_op_phase (struct list_head *peers, glusterd_op_t op, dict_t *op_ctx,
+                    dict_t *req_dict, char **op_errstr, int npeers)
+{
+        dict_t              *rsp_dict       = NULL;
+        int                 peer_cnt           = -1;
+        int                 ret             = -1;
+        char                *hostname       = NULL;
+        glusterd_peerinfo_t *peerinfo       = NULL;
+        xlator_t            *this           = NULL;
+        uuid_t              tmp_uuid        = {0};
+        char                *errstr         = NULL;
+        struct syncargs     args            = {0};
+
+        this = THIS;
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_op_commit_perform (op, req_dict, op_errstr, rsp_dict);
+        if (ret) {
+                hostname = "localhost";
+                goto commit_done;
+        }
+        if (op != GD_OP_SYNC_VOLUME) {
+                ret =  glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                "Failed to aggregate response "
+                                "from node/brick");
+                        goto out;
+                }
+        }
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+commit_done:
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_COMMIT_FAIL,
+                        gd_op_list[op], hostname, (*op_errstr) ? ":" : " ",
+                        (*op_errstr) ? *op_errstr : " ");
+                if (*op_errstr == NULL)
+                        gf_asprintf (op_errstr, OPERRSTR_COMMIT_FAIL,
+                                     hostname);
+                goto out;
+        }
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        gd_syncargs_init (&args, op_ctx);
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry (peerinfo, peers, op_peers_list) {
+                ret = gd_syncop_mgmt_commit_op (peerinfo->rpc, &args,
+                                                MY_UUID, tmp_uuid,
+                                                op, req_dict, op_ctx);
+                peer_cnt++;
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+        ret = args.op_ret;
+        if (args.errstr)
+                 *op_errstr = gf_strdup (args.errstr);
+        else if (dict_get_str (op_ctx, "errstr", &errstr) == 0)
+                *op_errstr = gf_strdup (errstr);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Sent commit op req for 'Volume %s' "
+                "to %d peers", gd_op_list[op], peer_cnt);
+out:
+        if (!ret)
+                glusterd_op_modify_op_ctx (op, op_ctx);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        GF_FREE (args.errstr);
+        args.errstr = NULL;
+
+        return ret;
+}
+
+int
+gd_unlock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, int op_ret,
+                    rpcsvc_request_t *req, dict_t *op_ctx, char *op_errstr,
+                    int npeers, char *volname, gf_boolean_t is_acquired,
+                    uuid_t txn_id)
+{
+        glusterd_peerinfo_t *peerinfo   = NULL;
+        glusterd_peerinfo_t *tmp        = NULL;
+        uuid_t              tmp_uuid    = {0};
+        int                 peer_cnt       = 0;
+        int                 ret         = -1;
+        xlator_t            *this       = NULL;
+        struct syncargs     args        = {0};
+        struct list_head   *peers       = NULL;
+
+        peers = &conf->xaction_peers;
+
+        if (!npeers) {
+                ret = 0;
+                goto out;
+        }
+
+        /* If the lock has not been held during this
+         * transaction, do not send unlock requests */
+        if (!is_acquired)
+                goto out;
+
+        this = THIS;
+        synctask_barrier_init((&args));
+        peer_cnt = 0;
+        list_for_each_entry_safe (peerinfo, tmp, peers, op_peers_list) {
+                if (conf->op_version < 3) {
+                        /* Only unlock peers that were locked */
+                        if (peerinfo->locked)
+                                gd_syncop_mgmt_unlock (peerinfo, &args,
+                                                       MY_UUID, tmp_uuid);
+                } else
+                        gd_syncop_mgmt_v3_unlock (op_ctx, peerinfo,
+                                                  &args, MY_UUID,
+                                                  tmp_uuid, txn_id);
+                peer_cnt++;
+                list_del_init (&peerinfo->op_peers_list);
+        }
+        gd_synctask_barrier_wait((&args), peer_cnt);
+        ret = args.op_ret;
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to unlock "
+                        "on some peer(s)");
+        }
+
+out:
+        glusterd_op_send_cli_response (op, op_ret, 0, req, op_ctx, op_errstr);
+        glusterd_op_clear_op (op);
+        if (is_acquired) {
+                /* Based on the op-version, we release *
+                 * the cluster or mgmt_v3 lock */
+                if (conf->op_version < 3)
+                        glusterd_unlock (MY_UUID);
+                else {
+                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
+                                                       "vol");
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to release lock for %s",
+                                        volname);
+                }
+        }
+
+        return 0;
+}
+
+int
+gd_get_brick_count (struct list_head *bricks)
+{
+        glusterd_pending_node_t *pending_node = NULL;
+        int                     npeers        = 0;
+        list_for_each_entry (pending_node, bricks, list) {
+                npeers++;
+        }
+        return npeers;
+}
+
+int
+gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                   char **op_errstr)
+{
+        glusterd_pending_node_t *pending_node = NULL;
+        struct list_head        selected = {0,};
+        xlator_t                *this = NULL;
+        int                     brick_count = 0;
+        int                     ret = -1;
+        rpc_clnt_t              *rpc = NULL;
+        dict_t                  *rsp_dict = NULL;
+        glusterd_conf_t         *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+        rsp_dict = dict_new ();
+        if (!rsp_dict) {
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&selected);
+        ret = glusterd_op_bricks_select (op, req_dict, op_errstr, &selected, rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                       (*op_errstr)? *op_errstr: "Brick op failed. Check "
+                       "glusterd log file for more details.");
+                goto out;
+        }
+
+        if (op == GD_OP_HEAL_VOLUME) {
+                ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict);
+                if (ret)
+                        goto out;
+        }
+        dict_unref (rsp_dict);
+        rsp_dict = NULL;
+
+        brick_count = 0;
+        list_for_each_entry (pending_node, &selected, list) {
+                rpc = glusterd_pending_node_get_rpc (pending_node);
+                if (!rpc) {
+                        if (pending_node->type == GD_NODE_REBALANCE) {
+                                ret = 0;
+                                glusterd_defrag_volume_node_rsp (req_dict,
+                                                                 NULL, op_ctx);
+                                goto out;
+                        }
+
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR, "Brick Op failed "
+                                "due to rpc failure.");
+                        goto out;
+                }
+                ret = gd_syncop_mgmt_brick_op (rpc, pending_node, op, req_dict,
+                                               op_ctx, op_errstr);
+                if (ret)
+                        goto out;
+
+                brick_count++;
+        }
+
+        ret = 0;
+out:
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+        gf_log (this->name, GF_LOG_DEBUG, "Sent op req to %d bricks",
+                brick_count);
+        return ret;
+}
+
+void
+gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
+{
+        int                         ret              = -1;
+        int                         npeers           = 0;
+        dict_t                      *req_dict        = NULL;
+        glusterd_conf_t             *conf            = NULL;
+        glusterd_op_t               op               = 0;
+        int32_t                     tmp_op           = 0;
+        char                        *op_errstr       = NULL;
+        char                        *tmp             = NULL;
+        char                        *volname         = NULL;
+        xlator_t                    *this            = NULL;
+        gf_boolean_t                is_acquired      = _gf_false;
+        uuid_t                      *txn_id          = NULL;
+        uuid_t                      *originator_uuid = NULL;
+        glusterd_op_info_t          txn_opinfo;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        /* Generate a transaction-id for this operation and
+         * save it in the dict */
+        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!txn_id) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_generate (*txn_id);
+
+        ret = dict_set_bin (op_ctx, "transaction_id",
+                            txn_id, sizeof(*txn_id));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set transaction id.");
+                goto out;
+        } else
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Transaction_id = %s", uuid_utoa (*txn_id));
+
+        /* Save the MY_UUID as the originator_uuid */
+        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
+                                     gf_common_mt_uuid_t);
+        if (!originator_uuid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*originator_uuid, MY_UUID);
+        ret = dict_set_bin (op_ctx, "originator_uuid",
+                            originator_uuid, sizeof (uuid_t));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to set originator_uuid.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume "
+                        "operation");
+                goto out;
+        }
+
+        op = tmp_op;
+
+        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+        if (conf->op_version < 3) {
+                ret = glusterd_lock (MY_UUID);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock");
+                        gf_asprintf (&op_errstr,
+                                     "Another transaction is in progress. "
+                                     "Please try again after sometime.");
+                        goto out;
+                }
+        } else {
+
+                /* If no volname is given as a part of the command, locks will
+                 * not be held */
+                ret = dict_get_str (op_ctx, "volname", &tmp);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG, "Failed to get volume "
+                                "name");
+                        goto local_locking_done;
+                } else {
+                        /* Use a copy of volname, as cli response will be
+                         * sent before the unlock, and the volname in the
+                         * dict, might be removed */
+                        volname = gf_strdup (tmp);
+                        if (!volname)
+                                goto out;
+                }
+
+                ret = glusterd_mgmt_v3_lock (volname, MY_UUID, "vol");
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to acquire lock for %s", volname);
+                        gf_asprintf (&op_errstr,
+                                     "Another transaction is in progress "
+                                     "for %s. Please try again after sometime.",
+                                     volname);
+                        goto out;
+                }
+        }
+
+        is_acquired = _gf_true;
+
+local_locking_done:
+
+        /* Save opinfo for this transaction with the transaction id */
+        glusterd_txn_opinfo_init (&txn_opinfo, NULL, &op, NULL, NULL);
+        ret = glusterd_set_txn_opinfo (txn_id, &txn_opinfo);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set transaction's opinfo");
+
+        opinfo = txn_opinfo;
+
+        INIT_LIST_HEAD (&conf->xaction_peers);
+
+        /* Make 'volume status tasks' command a local operation.
+         * This is accomplished by setting npeers to 0.
+         */
+        if (!glusterd_is_status_tasks_op (op, op_ctx))
+                npeers = gd_build_peers_list  (&conf->peers,
+                                               &conf->xaction_peers, op);
+
+        /* If no volname is given as a part of the command, locks will
+         * not be held */
+        if (volname) {
+                ret = gd_lock_op_phase (conf, op, op_ctx, &op_errstr, npeers, *txn_id);
+                if (ret)
+                        goto out;
+        }
+
+        ret = glusterd_op_build_payload (&req_dict, &op_errstr, op_ctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
+                        gd_op_list[op]);
+                if (op_errstr == NULL)
+                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+                goto out;
+        }
+
+        ret = gd_stage_op_phase (&conf->xaction_peers, op, op_ctx, req_dict,
+                                 &op_errstr, npeers);
+        if (ret)
+                goto out;
+
+        ret = gd_brick_op_phase (op, op_ctx, req_dict, &op_errstr);
+        if (ret)
+                goto out;
+
+        ret = gd_commit_op_phase (&conf->xaction_peers, op, op_ctx, req_dict,
+                                  &op_errstr, npeers);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        (void) gd_unlock_op_phase (conf, op, ret, req, op_ctx, op_errstr,
+                                   npeers, volname, is_acquired, *txn_id);
+
+        /* Clearing the transaction opinfo */
+        ret = glusterd_clear_txn_opinfo (txn_id);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to clear transaction's opinfo");
+
+        if (volname)
+                GF_FREE (volname);
+
+        if (req_dict)
+                dict_unref (req_dict);
+
+        if (op_errstr) {
+                GF_FREE (op_errstr);
+                op_errstr = NULL;
+        }
+
+        return;
+}
+
+int32_t
+glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
+                            void *dict)
+{
+        int              ret = 0;
+
+        ret = dict_set_int32 (dict, GD_SYNC_OPCODE_KEY, op);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "dict set failed for setting operations");
+                goto out;
+        }
+
+        gd_sync_task_begin (dict, req);
+        ret = 0;
+out:
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h
new file mode 100644
index 000000000..e83ea2f4c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h
@@ -0,0 +1,71 @@
+/*
+   Copyright (c) 2012-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __RPC_SYNCOP_H
+#define __RPC_SYNCOP_H
+
+#include "syncop.h"
+#include "glusterd-sm.h"
+#include "glusterd.h"
+
+#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
+
+/* gd_syncop_* */
+#define GD_SYNCOP(rpc, stb, cookie, cbk, req, prog, procnum, xdrproc) do {     \
+                int ret = 0;                                                   \
+                struct  synctask        *task = NULL;                          \
+                glusterd_conf_t         *conf= THIS->private;                  \
+                                                                               \
+                task = synctask_get ();                                        \
+                stb->task = task;                                              \
+                                                                               \
+                /*This is to ensure that the brick_op_cbk is able to           \
+                 * take the big lock*/                                         \
+                synclock_unlock (&conf->big_lock);                             \
+                ret = gd_syncop_submit_request (rpc, req, stb, cookie,         \
+                                                prog, procnum, cbk,            \
+                                                (xdrproc_t)xdrproc);           \
+                if (!ret)                                                      \
+                        synctask_yield (stb->task);                            \
+                synclock_lock (&conf->big_lock);                               \
+        } while (0)
+
+
+int gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
+                              void *cookie, rpc_clnt_prog_t *prog, int procnum,
+                              fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+
+
+int gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                         uuid_t my_uuid, uuid_t recv_uuid);
+int gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                           uuid_t my_uuid, uuid_t recv_uuid);
+int gd_syncop_mgmt_stage_op (struct rpc_clnt *rpc, struct syncargs *arg,
+                             uuid_t my_uuid, uuid_t recv_uuid, int op,
+                             dict_t *dict_out, dict_t *op_ctx);
+int gd_syncop_mgmt_commit_op (struct rpc_clnt *rpc, struct syncargs *arg,
+                              uuid_t my_uuid, uuid_t recv_uuid, int op,
+                              dict_t *dict_out, dict_t *op_ctx);
+
+void
+gd_synctask_barrier_wait (struct syncargs *args, int count);
+
+int
+gd_build_peers_list (struct list_head *peers, struct list_head *xact_peers,
+                     glusterd_op_t op);
+int
+gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                   char **op_errstr);
+
+int
+glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp);
+
+void
+gd_syncargs_init (struct syncargs *args, dict_t *op_ctx);
+#endif /* __RPC_SYNCOP_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
new file mode 100644
index 000000000..e8ae05851
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -0,0 +1,8894 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <inttypes.h>
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "compat.h"
+#include "dict.h"
+#include "xlator.h"
+#include "logging.h"
+#include "timer.h"
+#include "defaults.h"
+#include "compat.h"
+#include "syncop.h"
+#include "run.h"
+#include "compat-errno.h"
+#include "statedump.h"
+#include "syscall.h"
+#include "glusterd-mem-types.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-sm.h"
+#include "glusterd-utils.h"
+#include "glusterd-store.h"
+#include "glusterd-volgen.h"
+#include "glusterd-pmap.h"
+#include "glusterfs-acl.h"
+#include "glusterd-locks.h"
+
+#include "xdr-generic.h"
+#include <sys/resource.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <sys/types.h>
+#include <net/if.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <rpc/pmap_clnt.h>
+#include <unistd.h>
+#include <fnmatch.h>
+#include <sys/statvfs.h>
+#include <ifaddrs.h>
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
+
+#ifdef GF_LINUX_HOST_OS
+#include <mntent.h>
+#endif
+
+#ifdef GF_SOLARIS_HOST_OS
+#include <sys/sockio.h>
+#endif
+
+#define NFS_PROGRAM         100003
+#define NFSV3_VERSION       3
+
+#define MOUNT_PROGRAM       100005
+#define MOUNTV3_VERSION     3
+#define MOUNTV1_VERSION     1
+
+#define NLM_PROGRAM         100021
+#define NLMV4_VERSION       4
+#define NLMV1_VERSION       1
+
+#define CEILING_POS(X) (((X)-(int)(X)) > 0 ? (int)((X)+1) : (int)(X))
+
+static glusterd_lock_t lock;
+
+char*
+gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo)
+{
+        if ((peerinfo == NULL) || uuid_is_null (peerinfo->uuid))
+                return NULL;
+
+        if (peerinfo->uuid_str[0] == '\0')
+                uuid_utoa_r (peerinfo->uuid, peerinfo->uuid_str);
+
+        return peerinfo->uuid_str;
+}
+
+
+int32_t
+glusterd_get_lock_owner (uuid_t *uuid)
+{
+        uuid_copy (*uuid, lock.owner) ;
+        return 0;
+}
+
+static int32_t
+glusterd_set_lock_owner (uuid_t owner)
+{
+        uuid_copy (lock.owner, owner);
+        //TODO: set timestamp
+        return 0;
+}
+
+static int32_t
+glusterd_unset_lock_owner (uuid_t owner)
+{
+        uuid_clear (lock.owner);
+        //TODO: set timestamp
+        return 0;
+}
+
+gf_boolean_t
+glusterd_is_fuse_available ()
+{
+
+        int     fd = 0;
+
+        fd = open ("/dev/fuse", O_RDWR);
+
+        if (fd > -1 && !close (fd))
+                return _gf_true;
+        else
+                return _gf_false;
+}
+
+int32_t
+glusterd_lock (uuid_t   uuid)
+{
+
+        uuid_t  owner;
+        char    new_owner_str[50];
+        char    owner_str[50];
+        int     ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (uuid);
+
+        glusterd_get_lock_owner (&owner);
+
+        if (!uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get lock"
+                        " for uuid: %s, lock held by: %s",
+                        uuid_utoa_r (uuid, new_owner_str),
+                        uuid_utoa_r (owner, owner_str));
+                goto out;
+        }
+
+        ret = glusterd_set_lock_owner (uuid);
+
+        if (!ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Cluster lock held by"
+                         " %s", uuid_utoa (uuid));
+        }
+
+out:
+        return ret;
+}
+
+
+int32_t
+glusterd_unlock (uuid_t uuid)
+{
+        uuid_t  owner;
+        char    new_owner_str[50];
+        char    owner_str[50];
+        int32_t ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (uuid);
+
+        glusterd_get_lock_owner (&owner);
+
+        if (uuid_is_null (owner)) {
+                gf_log (this->name, GF_LOG_ERROR, "Cluster lock not held!");
+                goto out;
+        }
+
+        ret = uuid_compare (uuid, owner);
+
+        if (ret) {
+               gf_log (this->name, GF_LOG_ERROR, "Cluster lock held by %s ,"
+                       "unlock req from %s!", uuid_utoa_r (owner ,owner_str)
+                        , uuid_utoa_r (uuid, new_owner_str));
+               goto out;
+        }
+
+        ret = glusterd_unset_lock_owner (uuid);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to clear cluster "
+                        "lock");
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+int
+glusterd_get_uuid (uuid_t *uuid)
+{
+        glusterd_conf_t         *priv = NULL;
+
+        priv = THIS->private;
+
+        GF_ASSERT (priv);
+
+        uuid_copy (*uuid, MY_UUID);
+
+        return 0;
+}
+
+int
+glusterd_submit_request (struct rpc_clnt *rpc, void *req,
+                         call_frame_t *frame, rpc_clnt_prog_t *prog,
+                         int procnum, struct iobref *iobref,
+                         xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+        int                     ret         = -1;
+        struct iobuf            *iobuf      = NULL;
+        int                     count      = 0;
+        char                    new_iobref = 0, start_ping = 0;
+        struct iovec            iov         = {0, };
+        ssize_t                 req_size    = 0;
+
+        GF_ASSERT (rpc);
+        GF_ASSERT (this);
+
+        if (req) {
+                req_size = xdr_sizeof (xdrproc, req);
+                iobuf = iobuf_get2 (this->ctx->iobuf_pool, req_size);
+                if (!iobuf) {
+                        goto out;
+                };
+
+                if (!iobref) {
+                        iobref = iobref_new ();
+                        if (!iobref) {
+                                goto out;
+                        }
+
+                        new_iobref = 1;
+                }
+
+                iobref_add (iobref, iobuf);
+
+                iov.iov_base = iobuf->ptr;
+                iov.iov_len  = iobuf_pagesize (iobuf);
+
+                /* Create the xdr payload */
+                ret = xdr_serialize_generic (iov, req, xdrproc);
+                if (ret == -1) {
+                        goto out;
+                }
+                iov.iov_len = ret;
+                count = 1;
+        }
+
+        /* Send the msg */
+        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn,
+                               &iov, count,
+                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+
+        if (ret == 0) {
+                pthread_mutex_lock (&rpc->conn.lock);
+                {
+                        if (!rpc->conn.ping_started) {
+                                start_ping = 1;
+                        }
+                }
+                pthread_mutex_unlock (&rpc->conn.lock);
+        }
+
+        if (start_ping)
+                //client_start_ping ((void *) this);
+
+        ret = 0;
+out:
+        if (new_iobref) {
+                iobref_unref (iobref);
+        }
+
+        iobuf_unref (iobuf);
+
+        return ret;
+}
+
+struct iobuf *
+glusterd_serialize_reply (rpcsvc_request_t *req, void *arg,
+                          struct iovec *outmsg, xdrproc_t xdrproc)
+{
+        struct iobuf            *iob = NULL;
+        ssize_t                  retlen = -1;
+        ssize_t                  rsp_size = 0;
+
+        /* First, get the io buffer into which the reply in arg will
+         * be serialized.
+         */
+        rsp_size = xdr_sizeof (xdrproc, arg);
+        iob = iobuf_get2 (req->svc->ctx->iobuf_pool, rsp_size);
+        if (!iob) {
+                gf_log ("", GF_LOG_ERROR, "Failed to get iobuf");
+                goto ret;
+        }
+
+        iobuf_to_iovec (iob, outmsg);
+        /* Use the given serializer to translate the give C structure in arg
+         * to XDR format which will be written into the buffer in outmsg.
+         */
+        /* retlen is used to received the error since size_t is unsigned and we
+         * need -1 for error notification during encoding.
+         */
+        retlen = xdr_serialize_generic (*outmsg, arg, xdrproc);
+        if (retlen == -1) {
+                gf_log ("", GF_LOG_ERROR, "Failed to encode message");
+                goto ret;
+        }
+
+        outmsg->iov_len = retlen;
+ret:
+        if (retlen == -1) {
+                iobuf_unref (iob);
+                iob = NULL;
+        }
+
+        return iob;
+}
+
+int
+glusterd_submit_reply (rpcsvc_request_t *req, void *arg,
+                       struct iovec *payload, int payloadcount,
+                       struct iobref *iobref, xdrproc_t xdrproc)
+{
+        struct iobuf           *iob        = NULL;
+        int                     ret        = -1;
+        struct iovec            rsp        = {0,};
+        char                    new_iobref = 0;
+
+        if (!req) {
+                GF_ASSERT (req);
+                goto out;
+        }
+
+        if (!iobref) {
+                iobref = iobref_new ();
+                if (!iobref) {
+                        gf_log ("", GF_LOG_ERROR, "out of memory");
+                        goto out;
+                }
+
+                new_iobref = 1;
+        }
+
+        iob = glusterd_serialize_reply (req, arg, &rsp, xdrproc);
+        if (!iob) {
+                gf_log ("", GF_LOG_ERROR, "Failed to serialize reply");
+        } else {
+                iobref_add (iobref, iob);
+        }
+
+        ret = rpcsvc_submit_generic (req, &rsp, 1, payload, payloadcount,
+                                     iobref);
+
+        /* Now that we've done our job of handing the message to the RPC layer
+         * we can safely unref the iob in the hope that RPC layer must have
+         * ref'ed the iob on receiving into the txlist.
+         */
+        if (ret == -1) {
+                gf_log ("", GF_LOG_ERROR, "Reply submission failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        if (new_iobref) {
+                iobref_unref (iobref);
+        }
+
+        if (iob)
+                iobuf_unref (iob);
+        return ret;
+}
+
+gf_boolean_t
+glusterd_check_volume_exists (char *volname)
+{
+        char pathname[1024] = {0,};
+        struct stat stbuf = {0,};
+        int32_t ret = -1;
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+
+        snprintf (pathname, 1024, "%s/vols/%s", priv->workdir,
+                  volname);
+
+        ret = stat (pathname, &stbuf);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "Volume %s does not exist."
+                        "stat failed with errno : %d on path: %s",
+                        volname, errno, pathname);
+                return _gf_false;
+        }
+
+        return _gf_true;
+}
+
+int32_t
+glusterd_volinfo_new (glusterd_volinfo_t **volinfo)
+{
+        glusterd_volinfo_t      *new_volinfo = NULL;
+        int32_t                 ret = -1;
+
+        GF_ASSERT (volinfo);
+
+        new_volinfo = GF_CALLOC (1, sizeof(*new_volinfo),
+                                 gf_gld_mt_glusterd_volinfo_t);
+
+        if (!new_volinfo)
+                goto out;
+
+        LOCK_INIT (&new_volinfo->lock);
+        INIT_LIST_HEAD (&new_volinfo->vol_list);
+        INIT_LIST_HEAD (&new_volinfo->snapvol_list);
+        INIT_LIST_HEAD (&new_volinfo->bricks);
+        INIT_LIST_HEAD (&new_volinfo->snap_volumes);
+
+        new_volinfo->dict = dict_new ();
+        if (!new_volinfo->dict) {
+                GF_FREE (new_volinfo);
+
+                goto out;
+        }
+
+        new_volinfo->gsync_slaves = dict_new ();
+        if (!new_volinfo->gsync_slaves) {
+                GF_FREE (new_volinfo);
+
+                goto out;
+        }
+
+        snprintf (new_volinfo->parent_volname, GLUSTERD_MAX_VOLUME_NAME, "N/A");
+
+        new_volinfo->snap_max_hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
+        new_volinfo->xl = THIS;
+
+        *volinfo = new_volinfo;
+
+        ret = 0;
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* This function will create a new volinfo and then
+ * dup the entries from volinfo to the new_volinfo.
+ *
+ * @param volinfo       volinfo which will be duplicated
+ * @param dup_volinfo   new volinfo which will be created
+ * @param set_userauth  if this true then auth info is also set
+ *
+ * @return 0 on success else -1
+ */
+int32_t
+glusterd_volinfo_dup (glusterd_volinfo_t *volinfo,
+                      glusterd_volinfo_t **dup_volinfo,
+                      gf_boolean_t set_userauth)
+{
+        int32_t                 ret             = -1;
+        xlator_t                *this           = NULL;
+        glusterd_volinfo_t      *new_volinfo    = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, dup_volinfo, out);
+
+        ret = glusterd_volinfo_new (&new_volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "not able to create the "
+                        "duplicate volinfo for the volume %s",
+                        volinfo->volname);
+                goto out;
+        }
+
+        new_volinfo->type = volinfo->type;
+        new_volinfo->replica_count = volinfo->replica_count;
+        new_volinfo->stripe_count = volinfo->stripe_count;
+        new_volinfo->dist_leaf_count = volinfo->dist_leaf_count;
+        new_volinfo->sub_count = volinfo->sub_count;
+        new_volinfo->transport_type = volinfo->transport_type;
+        new_volinfo->nfs_transport_type = volinfo->nfs_transport_type;
+        new_volinfo->brick_count = volinfo->brick_count;
+
+        dict_copy (volinfo->dict, new_volinfo->dict);
+        gd_update_volume_op_versions (new_volinfo);
+
+        if (set_userauth) {
+                glusterd_auth_set_username (new_volinfo,
+                                            volinfo->auth.username);
+                glusterd_auth_set_password (new_volinfo,
+                                            volinfo->auth.password);
+        }
+
+        *dup_volinfo = new_volinfo;
+        ret = 0;
+out:
+        if (ret && (NULL != new_volinfo)) {
+                (void) glusterd_volinfo_delete (new_volinfo);
+        }
+        return ret;
+}
+
+/* This function will duplicate brickinfo
+ *
+ * @param brickinfo     Source brickinfo
+ * @param dup_brickinfo Destination brickinfo
+ *
+ * @return 0 on success else -1
+ */
+int32_t
+glusterd_brickinfo_dup (glusterd_brickinfo_t *brickinfo,
+                        glusterd_brickinfo_t *dup_brickinfo)
+{
+        int32_t         ret     = -1;
+        xlator_t        *this   = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, brickinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, dup_brickinfo, out);
+
+        strcpy (dup_brickinfo->hostname, brickinfo->hostname);
+        strcpy (dup_brickinfo->path, brickinfo->path);
+        strcpy (dup_brickinfo->device_path, brickinfo->device_path);
+        ret = gf_canonicalize_path (dup_brickinfo->path);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to canonicalize "
+                        "brick path");
+                goto out;
+        }
+        uuid_copy (dup_brickinfo->uuid, brickinfo->uuid);
+
+        dup_brickinfo->port = brickinfo->port;
+        dup_brickinfo->rdma_port = brickinfo->rdma_port;
+        if (NULL != brickinfo->logfile) {
+                dup_brickinfo->logfile = gf_strdup (brickinfo->logfile);
+                if (NULL == dup_brickinfo->logfile) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        dup_brickinfo->status = brickinfo->status;
+        dup_brickinfo->snap_status = brickinfo->snap_status;
+out:
+        return ret;
+}
+
+/* This function will copy snap volinfo to the new
+ * passed volinfo and regenerate backend store files
+ * for the restored snap.
+ *
+ * @param new_volinfo   new volinfo
+ * @param snap_volinfo  volinfo of snap volume
+ *
+ * @return 0 on success and -1 on failure
+ *
+ * TODO: Duplicate all members of volinfo, e.g. geo-rep sync slaves
+ */
+int32_t
+glusterd_snap_volinfo_restore (dict_t *rsp_dict,
+                               glusterd_volinfo_t *new_volinfo,
+                               glusterd_volinfo_t *snap_volinfo)
+{
+        int32_t                 brick_count     = -1;
+        int32_t                 ret             = -1;
+        xlator_t                *this           = NULL;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        glusterd_brickinfo_t    *new_brickinfo  = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+
+        GF_VALIDATE_OR_GOTO (this->name, new_volinfo, out);
+        GF_VALIDATE_OR_GOTO (this->name, snap_volinfo, out);
+
+        brick_count = 0;
+        list_for_each_entry (brickinfo, &snap_volinfo->bricks, brick_list) {
+                ret = glusterd_brickinfo_new (&new_brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to create "
+                                "new brickinfo");
+                        goto out;
+                }
+
+                /* Duplicate brickinfo */
+                ret = glusterd_brickinfo_dup (brickinfo, new_brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to dup "
+                                "brickinfo");
+                        goto out;
+                }
+
+                /* If the brick is not of this peer, or snapshot is missed *
+                 * for the brick do not replace the xattr for it */
+                if ((!uuid_compare (brickinfo->uuid, MY_UUID)) &&
+                    (brickinfo->snap_status != -1)) {
+                        /* We need to replace the volume id of all the bricks
+                         * to the volume id of the origin volume. new_volinfo
+                         * has the origin volume's volume id*/
+                        ret = sys_lsetxattr (new_brickinfo->path,
+                                             GF_XATTR_VOL_ID_KEY,
+                                             new_volinfo->volume_id,
+                                             sizeof (new_volinfo->volume_id),
+                                             XATTR_REPLACE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "set extended attribute %s on %s. "
+                                        "Reason: %s, snap: %s",
+                                        GF_XATTR_VOL_ID_KEY,
+                                        new_brickinfo->path, strerror (errno),
+                                        new_volinfo->volname);
+                                goto out;
+                        }
+                }
+
+                /* If a snapshot is pending for this brick then
+                 * restore should also be pending
+                 */
+                if (brickinfo->snap_status == -1) {
+                        /* Adding missed delete to the dict */
+                        ret = glusterd_add_missed_snaps_to_dict
+                                                (rsp_dict,
+                                                 snap_volinfo->volname,
+                                                 brickinfo,
+                                                 brick_count + 1,
+                                                 GF_SNAP_OPTION_TYPE_RESTORE);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to add missed snapshot info "
+                                        "for %s:%s in the rsp_dict",
+                                        brickinfo->hostname,
+                                        brickinfo->path);
+                                goto out;
+                        }
+                }
+
+                list_add_tail (&new_brickinfo->brick_list,
+                                &new_volinfo->bricks);
+                /* ownership of new_brickinfo is passed to new_volinfo */
+                new_brickinfo = NULL;
+                brick_count++;
+        }
+
+        /* Regenerate all volfiles */
+        ret = glusterd_create_volfiles_and_notify_services (new_volinfo);
+
+out:
+        if (ret && (NULL != new_brickinfo)) {
+                (void) glusterd_brickinfo_delete (new_brickinfo);
+        }
+
+        return ret;
+}
+
+void
+glusterd_auth_cleanup (glusterd_volinfo_t *volinfo) {
+
+        GF_ASSERT (volinfo);
+
+        GF_FREE (volinfo->auth.username);
+
+        GF_FREE (volinfo->auth.password);
+}
+
+char *
+glusterd_auth_get_username (glusterd_volinfo_t *volinfo) {
+
+        GF_ASSERT (volinfo);
+
+        return volinfo->auth.username;
+}
+
+char *
+glusterd_auth_get_password (glusterd_volinfo_t *volinfo) {
+
+        GF_ASSERT (volinfo);
+
+        return volinfo->auth.password;
+}
+
+int32_t
+glusterd_auth_set_username (glusterd_volinfo_t *volinfo, char *username) {
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (username);
+
+        volinfo->auth.username = gf_strdup (username);
+        return 0;
+}
+
+int32_t
+glusterd_auth_set_password (glusterd_volinfo_t *volinfo, char *password) {
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (password);
+
+        volinfo->auth.password = gf_strdup (password);
+        return 0;
+}
+
+int32_t
+glusterd_brickinfo_delete (glusterd_brickinfo_t *brickinfo)
+{
+        int32_t         ret = -1;
+
+        GF_ASSERT (brickinfo);
+
+        list_del_init (&brickinfo->brick_list);
+
+        GF_FREE (brickinfo->logfile);
+        GF_FREE (brickinfo);
+
+        ret = 0;
+
+        return ret;
+}
+
+int32_t
+glusterd_volume_brickinfos_delete (glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_brickinfo_t    *tmp = NULL;
+        int32_t                 ret = 0;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
+                                   brick_list) {
+                ret = glusterd_brickinfo_delete (brickinfo);
+                if (ret)
+                        goto out;
+        }
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_volinfo_delete (glusterd_volinfo_t *volinfo)
+{
+        int32_t                 ret = -1;
+
+        GF_ASSERT (volinfo);
+
+        list_del_init (&volinfo->vol_list);
+        list_del_init (&volinfo->snapvol_list);
+
+        ret = glusterd_volume_brickinfos_delete (volinfo);
+        if (ret)
+                goto out;
+        if (volinfo->dict)
+                dict_unref (volinfo->dict);
+        if (volinfo->gsync_slaves)
+                dict_unref (volinfo->gsync_slaves);
+        GF_FREE (volinfo->logdir);
+
+        glusterd_auth_cleanup (volinfo);
+
+        GF_FREE (volinfo);
+        ret = 0;
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo)
+{
+        glusterd_brickinfo_t      *new_brickinfo = NULL;
+        int32_t                   ret = -1;
+
+        GF_ASSERT (brickinfo);
+
+        new_brickinfo = GF_CALLOC (1, sizeof(*new_brickinfo),
+                                   gf_gld_mt_glusterd_brickinfo_t);
+
+        if (!new_brickinfo)
+                goto out;
+
+        INIT_LIST_HEAD (&new_brickinfo->brick_list);
+
+        *brickinfo = new_brickinfo;
+
+        ret = 0;
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo)
+{
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (brickinfo);
+
+        ret = glusterd_hostname_to_uuid (brickinfo->hostname, brickinfo->uuid);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_brickinfo_new_from_brick (char *brick,
+                                   glusterd_brickinfo_t **brickinfo)
+{
+        int32_t                 ret = -1;
+        glusterd_brickinfo_t    *new_brickinfo = NULL;
+        char                    *hostname = NULL;
+        char                    *path = NULL;
+        char                    *tmp_host = NULL;
+        char                    *tmp_path = NULL;
+        char                    *vg       = NULL;
+
+        GF_ASSERT (brick);
+        GF_ASSERT (brickinfo);
+
+        tmp_host = gf_strdup (brick);
+        if (tmp_host && !get_host_name (tmp_host, &hostname))
+                goto out;
+        tmp_path = gf_strdup (brick);
+        if (tmp_path && !get_path_name (tmp_path, &path))
+                goto out;
+
+        GF_ASSERT (hostname);
+        GF_ASSERT (path);
+
+        ret = glusterd_brickinfo_new (&new_brickinfo);
+        if (ret)
+                goto out;
+
+#ifdef HAVE_BD_XLATOR
+        vg = strchr (path, '?');
+        /* ? is used as a delimiter for vg */
+        if (vg) {
+                strncpy (new_brickinfo->vg, vg + 1, PATH_MAX - 1);
+                *vg = '\0';
+        }
+        new_brickinfo->caps = CAPS_BD;
+#else
+        vg = NULL; /* Avoid compiler warnings when BD not enabled */
+#endif
+        ret = gf_canonicalize_path (path);
+        if (ret)
+                goto out;
+
+        strncpy (new_brickinfo->hostname, hostname, 1024);
+        strncpy (new_brickinfo->path, path, 1024);
+
+        *brickinfo = new_brickinfo;
+
+        ret = 0;
+out:
+        GF_FREE (tmp_host);
+        if (tmp_host)
+                GF_FREE (tmp_path);
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static gf_boolean_t
+_is_prefix (char *str1, char *str2)
+{
+        GF_ASSERT (str1);
+        GF_ASSERT (str2);
+
+        int             i = 0;
+        int             len1 = 0;
+        int             len2 = 0;
+        int             small_len = 0;
+        char            *bigger = NULL;
+        gf_boolean_t    prefix = _gf_true;
+
+        len1 = strlen (str1);
+        len2 = strlen (str2);
+        small_len = min (len1, len2);
+        for (i = 0; i < small_len; i++) {
+                if (str1[i] != str2[i]) {
+                        prefix = _gf_false;
+                        break;
+                }
+        }
+
+        if (len1 < len2)
+            bigger = str2;
+
+        else if (len1 > len2)
+            bigger = str1;
+
+        else
+            return prefix;
+
+        if (bigger[small_len] != '/')
+            prefix = _gf_false;
+
+        return prefix;
+}
+
+/* Checks if @path is available in the peer identified by @uuid
+ * 'availability' is determined by querying current state of volumes
+ * in the cluster. */
+gf_boolean_t
+glusterd_is_brickpath_available (uuid_t uuid, char *path)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_volinfo_t      *volinfo   = NULL;
+        glusterd_conf_t         *priv      = NULL;
+        gf_boolean_t            available  = _gf_false;
+        char                    tmp_path[PATH_MAX+1] = {0};
+        char                    tmp_brickpath[PATH_MAX+1] = {0};
+
+        priv = THIS->private;
+
+        strncpy (tmp_path, path, PATH_MAX);
+        /* path may not yet exist */
+        if (!realpath (path, tmp_path)) {
+                if (errno != ENOENT) {
+                        goto out;
+                }
+                /* When realpath(3) fails, tmp_path is undefined. */
+                strncpy(tmp_path,path,PATH_MAX);
+        }
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        if (uuid_compare (uuid, brickinfo->uuid))
+                                continue;
+
+                        if (!realpath (brickinfo->path, tmp_brickpath)) {
+                            if (errno == ENOENT)
+                                strncpy (tmp_brickpath, brickinfo->path,
+                                         PATH_MAX);
+                            else
+                                goto out;
+                        }
+
+                        if (_is_prefix (tmp_brickpath, tmp_path))
+                                goto out;
+                }
+        }
+        available = _gf_true;
+out:
+        return available;
+}
+
+#ifdef HAVE_BD_XLATOR
+/*
+ * Sets the tag of the format "trusted.glusterfs.volume-id:<uuid>" in
+ * the brick VG. It is used to avoid using same VG for another brick.
+ * @volume-id - gfid, @brick - brick info, @msg - Error message returned
+ * to the caller
+ */
+int
+glusterd_bd_set_vg_tag (unsigned char *volume_id, glusterd_brickinfo_t *brick,
+                        char *msg, int msg_size)
+{
+        lvm_t        handle    = NULL;
+        vg_t         vg        = NULL;
+        char        *uuid      = NULL;
+        int          ret       = -1;
+
+        gf_asprintf (&uuid, "%s:%s", GF_XATTR_VOL_ID_KEY,
+                     uuid_utoa (volume_id));
+        if (!uuid) {
+                snprintf (msg, sizeof(*msg), "Could not allocate memory "
+                          "for tag");
+                return -1;
+        }
+
+        handle = lvm_init (NULL);
+        if (!handle) {
+                snprintf (msg, sizeof(*msg), "lvm_init failed");
+                goto out;
+        }
+
+        vg = lvm_vg_open (handle, brick->vg, "w", 0);
+        if (!vg) {
+                snprintf (msg, sizeof(*msg), "Could not open VG %s",
+                          brick->vg);
+                goto out;
+        }
+
+        if (lvm_vg_add_tag (vg, uuid) < 0) {
+                snprintf (msg, sizeof(*msg), "Could not set tag %s for "
+                          "VG %s", uuid, brick->vg);
+                goto out;
+        }
+        lvm_vg_write (vg);
+        ret = 0;
+out:
+        GF_FREE (uuid);
+
+        if (vg)
+                lvm_vg_close (vg);
+        if (handle)
+                lvm_quit (handle);
+
+        return ret;
+}
+#endif
+
+int
+glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
+                                        uuid_t volume_id, char **op_errstr,
+                                        gf_boolean_t is_force)
+{
+        int          ret                 = -1;
+        char         parentdir[PATH_MAX] = {0,};
+        struct stat  parent_st           = {0,};
+        struct stat  brick_st            = {0,};
+        struct stat  root_st             = {0,};
+        char         msg[2048]           = {0,};
+        gf_boolean_t is_created          = _gf_false;
+
+        ret = mkdir (brickinfo->path, 0777);
+        if (ret) {
+                if (errno != EEXIST) {
+                        snprintf (msg, sizeof (msg), "Failed to create brick "
+                                  "directory for brick %s:%s. Reason : %s ",
+                                  brickinfo->hostname, brickinfo->path,
+                                  strerror (errno));
+                        goto out;
+                }
+        } else {
+                is_created = _gf_true;
+        }
+
+        ret = lstat (brickinfo->path, &brick_st);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "lstat failed on %s. Reason : %s",
+                          brickinfo->path, strerror (errno));
+                goto out;
+        }
+
+        if ((!is_created) && (!S_ISDIR (brick_st.st_mode))) {
+                snprintf (msg, sizeof (msg), "The provided path %s which is "
+                          "already present, is not a directory",
+                          brickinfo->path);
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (parentdir, sizeof (parentdir), "%s/..", brickinfo->path);
+
+        ret = lstat ("/", &root_st);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "lstat failed on /. Reason : %s",
+                          strerror (errno));
+                goto out;
+        }
+
+        ret = lstat (parentdir, &parent_st);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "lstat failed on %s. Reason : %s",
+                          parentdir, strerror (errno));
+                goto out;
+        }
+
+        if (!is_force) {
+                if (brick_st.st_dev != parent_st.st_dev) {
+                        snprintf (msg, sizeof (msg), "The brick %s:%s is a "
+                                  "mount point. Please create a sub-directory "
+                                  "under the mount point and use that as the "
+                                  "brick directory. Or use 'force' at the end "
+                                  "of the command if you want to override this "
+                                  "behavior.", brickinfo->hostname,
+                                  brickinfo->path);
+                        ret = -1;
+                        goto out;
+                }
+                else if (parent_st.st_dev == root_st.st_dev) {
+                        snprintf (msg, sizeof (msg), "The brick %s:%s is "
+                                  "is being created in the root partition. It "
+                                  "is recommended that you don't use the "
+                                  "system's root partition for storage backend."
+                                  " Or use 'force' at the end of the command if"
+                                  " you want to override this behavior.",
+                                  brickinfo->hostname, brickinfo->path);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+#ifdef HAVE_BD_XLATOR
+        if (brickinfo->vg[0]) {
+                ret = glusterd_bd_set_vg_tag (volume_id, brickinfo, msg,
+                                              sizeof(msg));
+                if (ret)
+                        goto out;
+        }
+#endif
+        ret = glusterd_check_and_set_brick_xattr (brickinfo->hostname,
+                                                  brickinfo->path, volume_id,
+                                                  op_errstr, is_force);
+        if (ret)
+                goto out;
+
+        ret = 0;
+
+out:
+        if (ret && is_created)
+                rmdir (brickinfo->path);
+        if (ret && !*op_errstr && msg[0] != '\0')
+                *op_errstr = gf_strdup (msg);
+
+        return ret;
+}
+
+int32_t
+glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
+                               glusterd_volinfo_t *volinfo,
+                               glusterd_brickinfo_t **brickinfo)
+{
+        glusterd_brickinfo_t    *brickiter = NULL;
+        uuid_t                  peer_uuid = {0};
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+
+        if (uuid) {
+                uuid_copy (peer_uuid, uuid);
+        } else {
+                ret = glusterd_hostname_to_uuid (hostname, peer_uuid);
+                if (ret)
+                        goto out;
+        }
+        ret = -1;
+        list_for_each_entry (brickiter, &volinfo->bricks, brick_list) {
+
+                if ((uuid_is_null (brickiter->uuid)) &&
+                    (glusterd_resolve_brick (brickiter) != 0))
+                        goto out;
+                if (uuid_compare (peer_uuid, brickiter->uuid))
+                        continue;
+
+                if (strcmp (brickiter->path, path) == 0) {
+                        gf_log (this->name, GF_LOG_DEBUG, LOGSTR_FOUND_BRICK,
+                                brickiter->hostname, brickiter->path,
+                                volinfo->volname);
+                        ret = 0;
+                        if (brickinfo)
+                                *brickinfo = brickiter;
+                        break;
+                }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_volume_brickinfo_get_by_brick (char *brick,
+                                        glusterd_volinfo_t *volinfo,
+                                        glusterd_brickinfo_t **brickinfo)
+{
+        int32_t                 ret = -1;
+        glusterd_brickinfo_t    *tmp_brickinfo = NULL;
+
+        GF_ASSERT (brick);
+        GF_ASSERT (volinfo);
+
+        ret = glusterd_brickinfo_new_from_brick (brick, &tmp_brickinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_volume_brickinfo_get (NULL, tmp_brickinfo->hostname,
+                                             tmp_brickinfo->path, volinfo,
+                                             brickinfo);
+        (void) glusterd_brickinfo_delete (tmp_brickinfo);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_brick_decommissioned (glusterd_volinfo_t *volinfo, char *hostname,
+                                  char *path)
+{
+        gf_boolean_t            decommissioned = _gf_false;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        int                     ret = -1;
+
+        ret = glusterd_volume_brickinfo_get (NULL, hostname, path, volinfo,
+                                             &brickinfo);
+        if (ret)
+                goto out;
+        decommissioned = brickinfo->decommissioned;
+out:
+        return decommissioned;
+}
+
+int32_t
+glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo)
+{
+        GF_ASSERT (peerinfo);
+        glusterd_peerctx_t      *peerctx = NULL;
+        gf_boolean_t            quorum_action = _gf_false;
+        glusterd_conf_t         *priv = THIS->private;
+
+        if (peerinfo->quorum_contrib != QUORUM_NONE)
+                quorum_action = _gf_true;
+        if (peerinfo->rpc) {
+                /* cleanup the saved-frames before last unref */
+                synclock_unlock (&priv->big_lock);
+                rpc_clnt_connection_cleanup (&peerinfo->rpc->conn);
+                synclock_lock (&priv->big_lock);
+
+                peerctx = peerinfo->rpc->mydata;
+                peerinfo->rpc->mydata = NULL;
+                peerinfo->rpc = rpc_clnt_unref (peerinfo->rpc);
+                peerinfo->rpc = NULL;
+                if (peerctx) {
+                        GF_FREE (peerctx->errstr);
+                        GF_FREE (peerctx);
+                }
+        }
+        glusterd_peer_destroy (peerinfo);
+
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        return 0;
+}
+
+int
+glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volinfo)
+{
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_volinfo_t      *voliter = NULL;
+        glusterd_conf_t         *priv = NULL;
+
+        if (!volume_id)
+                return -1;
+
+        this = THIS;
+        priv = this->private;
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (uuid_compare (volume_id, voliter->volume_id))
+                        continue;
+                *volinfo = voliter;
+                ret = 0;
+                gf_log (this->name, GF_LOG_DEBUG, "Volume %s found",
+                        voliter->volname);
+                break;
+        }
+        return ret;
+}
+
+int
+glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
+                                         glusterd_volinfo_t **volinfo)
+{
+        int32_t                  ret     = -1;
+        xlator_t                *this    = NULL;
+        glusterd_volinfo_t      *voliter = NULL;
+        glusterd_snap_t         *snap    = NULL;
+        glusterd_conf_t         *priv    = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (volinfo);
+
+        if (uuid_is_null(volume_id)) {
+                gf_log (this->name, GF_LOG_WARNING, "Volume UUID is NULL");
+                goto out;
+        }
+
+        list_for_each_entry (snap, &priv->snapshots, snap_list) {
+                list_for_each_entry (voliter, &snap->volumes, vol_list) {
+                        if (uuid_compare (volume_id, voliter->volume_id))
+                                continue;
+                        *volinfo = voliter;
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_WARNING, "Snap volume not found");
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
+{
+        glusterd_volinfo_t      *tmp_volinfo = NULL;
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+
+        GF_ASSERT (volname);
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (tmp_volinfo, &priv->volumes, vol_list) {
+                if (!strcmp (tmp_volinfo->volname, volname)) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Volume %s found",
+                                volname);
+                        ret = 0;
+                        *volinfo = tmp_volinfo;
+                        break;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snap_volinfo_find (char *snap_volname, glusterd_snap_t *snap,
+                            glusterd_volinfo_t **volinfo)
+{
+        int32_t                  ret         = -1;
+        xlator_t                *this        = NULL;
+        glusterd_volinfo_t      *snap_vol    = NULL;
+        glusterd_conf_t         *priv        = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+        GF_ASSERT (snap_volname);
+
+        list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
+                if (!strcmp (snap_vol->volname, snap_volname)) {
+                        ret = 0;
+                        *volinfo = snap_vol;
+                        goto out;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_WARNING, "Snap volume %s not found",
+                snap_volname);
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
+                                      glusterd_snap_t *snap,
+                                      glusterd_volinfo_t **volinfo)
+{
+        int32_t                  ret         = -1;
+        xlator_t                *this        = NULL;
+        glusterd_volinfo_t      *snap_vol    = NULL;
+        glusterd_conf_t         *priv        = NULL;
+
+        this = THIS;
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (snap);
+        GF_ASSERT (origin_volname);
+
+        list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
+                if (!strcmp (snap_vol->parent_volname, origin_volname)) {
+                        ret = 0;
+                        *volinfo = snap_vol;
+                        goto out;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Snap volume not found(snap: %s, "
+                "origin-volume: %s", snap->snapname, origin_volname);
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_service_stop (const char *service, char *pidfile, int sig,
+                       gf_boolean_t force_kill)
+{
+        int32_t  ret = -1;
+        pid_t    pid = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        if (!glusterd_is_service_running (pidfile, &pid)) {
+                ret = 0;
+                gf_log (this->name, GF_LOG_INFO, "%s already stopped", service);
+                goto out;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Stopping gluster %s running in pid: "
+                "%d", service, pid);
+
+        ret = kill (pid, sig);
+        if (!force_kill)
+                goto out;
+
+        sleep (1);
+        if (glusterd_is_service_running (pidfile, NULL)) {
+                ret = kill (pid, SIGKILL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Unable to "
+                                "kill pid %d reason: %s", pid,
+                                strerror(errno));
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len)
+{
+        char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};
+
+        md5_wrapper ((unsigned char *) sock_filepath, strlen(sock_filepath), md5_sum);
+        snprintf (sockpath, len, "%s/%s.socket", GLUSTERD_SOCK_DIR, md5_sum);
+}
+
+void
+glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,
+                                    glusterd_brickinfo_t *brickinfo,
+                                    char *sockpath, size_t len)
+{
+        char                    export_path[PATH_MAX] = {0,};
+        char                    sock_filepath[PATH_MAX] = {0,};
+        char                    volume_dir[PATH_MAX] = {0,};
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+        int                     expected_file_len = 0;
+
+        expected_file_len = strlen (GLUSTERD_SOCK_DIR) + strlen ("/") +
+                            MD5_DIGEST_LENGTH*2 + strlen (".socket") + 1;
+        GF_ASSERT (len >= expected_file_len);
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+
+        GLUSTERD_GET_VOLUME_DIR (volume_dir, volinfo, priv);
+        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, export_path);
+        snprintf (sock_filepath, PATH_MAX, "%s/run/%s-%s",
+                  volume_dir, brickinfo->hostname, export_path);
+
+        glusterd_set_socket_filepath (sock_filepath, sockpath, len);
+}
+
+/* connection happens only if it is not aleady connected,
+ * reconnections are taken care by rpc-layer
+ */
+int32_t
+glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
+                        glusterd_brickinfo_t  *brickinfo, char *socketpath)
+{
+        int                     ret = 0;
+        char                    volume_id_str[64];
+        char                    *brickid = NULL;
+        dict_t                  *options = NULL;
+        struct rpc_clnt         *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (socketpath);
+
+        if (brickinfo->rpc == NULL) {
+                /* Setting frame-timeout to 10mins (600seconds).
+                 * Unix domain sockets ensures that the connection is reliable.
+                 * The default timeout of 30mins used for unreliable network
+                 * connections is too long for unix domain socket connections.
+                 */
+                ret = rpc_transport_unix_options_build (&options, socketpath,
+                                                        600);
+                if (ret)
+                        goto out;
+
+                uuid_utoa_r (volinfo->volume_id, volume_id_str);
+                ret = gf_asprintf (&brickid, "%s:%s:%s", volume_id_str,
+                                   brickinfo->hostname, brickinfo->path);
+                if (ret < 0)
+                        goto out;
+
+                synclock_unlock (&priv->big_lock);
+                ret = glusterd_rpc_create (&rpc, options,
+                                           glusterd_brick_rpc_notify,
+                                           brickid);
+                synclock_lock (&priv->big_lock);
+                if (ret)
+                        goto out;
+                brickinfo->rpc = rpc;
+        }
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Caller should ensure that brick process is not running*/
+static void
+_reap_brick_process (char *pidfile, char *brickpath)
+{
+        unlink (pidfile);
+        /* Brick process is not running and pmap may have an entry for it.*/
+        pmap_registry_remove (THIS, 0, brickpath,
+                              GF_PMAP_PORT_BRICKSERVER, NULL);
+}
+
+static int
+_mk_rundir_p (glusterd_volinfo_t *volinfo)
+{
+        char voldir[PATH_MAX]   = {0,};
+        char rundir[PATH_MAX]   = {0,};
+        glusterd_conf_t *priv   = NULL;
+        xlator_t        *this   = NULL;
+        int             ret     = -1;
+
+        this = THIS;
+        priv = this->private;
+        GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, priv);
+        snprintf (rundir, sizeof (rundir)-1, "%s/run", voldir);
+        ret =  mkdir_p (rundir, 0777, _gf_true);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create rundir");
+        return ret;
+}
+
+int32_t
+glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
+                                 glusterd_brickinfo_t  *brickinfo,
+                                 gf_boolean_t wait)
+{
+        int32_t                 ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+        char                    pidfile[PATH_MAX+1] = {0,};
+        char                    volfile[PATH_MAX] = {0,};
+        runner_t                runner = {0,};
+        char                    exp_path[PATH_MAX] = {0,};
+        char                    logfile[PATH_MAX] = {0,};
+        int                     port = 0;
+        int                     rdma_port = 0;
+        char                    socketpath[PATH_MAX] = {0};
+        char                    glusterd_uuid[1024] = {0,};
+        char                    valgrind_logfile[PATH_MAX] = {0};
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (brickinfo->snap_status == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Snapshot is pending on %s:%s. "
+                        "Hence not starting the brick",
+                        brickinfo->hostname,
+                        brickinfo->path);
+                ret = 0;
+                goto out;
+        }
+
+        ret = _mk_rundir_p (volinfo);
+        if (ret)
+                goto out;
+
+        glusterd_set_brick_socket_filepath (volinfo, brickinfo, socketpath,
+                                            sizeof (socketpath));
+
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
+        if (glusterd_is_service_running (pidfile, NULL))
+                goto connect;
+
+        _reap_brick_process (pidfile, brickinfo->path);
+
+        port = brickinfo->port;
+        if (!port)
+                port = pmap_registry_alloc (THIS);
+
+        /* Build the exp_path, before starting the glusterfsd even in
+           valgrind mode. Otherwise all the glusterfsd processes start
+           writing the valgrind log to the same file.
+        */
+        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);
+        runinit (&runner);
+
+        if (priv->valgrind) {
+                /* Run bricks with valgrind */
+                if (volinfo->logdir) {
+                        snprintf (valgrind_logfile, PATH_MAX,
+                                  "%s/valgrind-%s-%s.log",
+                                  volinfo->logdir,
+                                  volinfo->volname, exp_path);
+                } else {
+                        snprintf (valgrind_logfile, PATH_MAX,
+                                  "%s/bricks/valgrind-%s-%s.log",
+                                  DEFAULT_LOG_FILE_DIRECTORY,
+                                  volinfo->volname, exp_path);
+                }
+
+                runner_add_args (&runner, "valgrind", "--leak-check=full",
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
+                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+        }
+
+        if (volinfo->is_snap_volume) {
+                snprintf (volfile, PATH_MAX,"/%s/%s/%s.%s.%s",
+                          GLUSTERD_VOL_SNAP_DIR_PREFIX,
+                          volinfo->snapshot->snapname, volinfo->volname,
+                          brickinfo->hostname, exp_path);
+        } else {
+                snprintf (volfile, PATH_MAX, "%s.%s.%s", volinfo->volname,
+                          brickinfo->hostname, exp_path);
+        }
+
+        if (volinfo->logdir) {
+                snprintf (logfile, PATH_MAX, "%s/%s.log",
+                          volinfo->logdir, exp_path);
+        } else {
+                snprintf (logfile, PATH_MAX, "%s/bricks/%s.log",
+                          DEFAULT_LOG_FILE_DIRECTORY, exp_path);
+        }
+        if (!brickinfo->logfile)
+                brickinfo->logfile = gf_strdup (logfile);
+
+        (void) snprintf (glusterd_uuid, 1024, "*-posix.glusterd-uuid=%s",
+                         uuid_utoa (MY_UUID));
+        runner_add_args (&runner, SBIN_DIR"/glusterfsd",
+                         "-s", brickinfo->hostname, "--volfile-id", volfile,
+                         "-p", pidfile, "-S", socketpath,
+                         "--brick-name", brickinfo->path,
+                         "-l", brickinfo->logfile,
+                         "--xlator-option", glusterd_uuid,
+                         NULL);
+
+        runner_add_arg (&runner, "--brick-port");
+        if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
+                runner_argprintf (&runner, "%d", port);
+        } else {
+                rdma_port = brickinfo->rdma_port;
+                if (!rdma_port)
+                        rdma_port = pmap_registry_alloc (THIS);
+                runner_argprintf (&runner, "%d,%d", port, rdma_port);
+                runner_add_arg (&runner, "--xlator-option");
+                runner_argprintf (&runner, "%s-server.transport.rdma.listen-port=%d",
+                                  volinfo->volname, rdma_port);
+        }
+
+        runner_add_arg (&runner, "--xlator-option");
+        runner_argprintf (&runner, "%s-server.listen-port=%d",
+                          volinfo->volname, port);
+
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
+        runner_log (&runner, "", GF_LOG_DEBUG, "Starting GlusterFS");
+        if (wait) {
+                synclock_unlock (&priv->big_lock);
+                ret = runner_run (&runner);
+                synclock_lock (&priv->big_lock);
+
+        } else {
+                ret = runner_run_nowait (&runner);
+        }
+
+        if (ret)
+                goto out;
+
+        brickinfo->port = port;
+        brickinfo->rdma_port = rdma_port;
+
+connect:
+        ret = glusterd_brick_connect (volinfo, brickinfo, socketpath);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to connect to brick %s:%s on %s",
+                        brickinfo->hostname, brickinfo->path, socketpath);
+                goto out;
+        }
+out:
+        return ret;
+}
+
+int32_t
+glusterd_brick_unlink_socket_file (glusterd_volinfo_t *volinfo,
+                                   glusterd_brickinfo_t *brickinfo)
+{
+        char                    path[PATH_MAX] = {0,};
+        char                    socketpath[PATH_MAX] = {0};
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+        int                     ret = 0;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+        glusterd_set_brick_socket_filepath (volinfo, brickinfo, socketpath,
+                                            sizeof (socketpath));
+        ret = unlink (socketpath);
+        if (ret && (ENOENT == errno)) {
+                ret = 0;
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to remove %s"
+                        " error: %s", socketpath, strerror (errno));
+        }
+
+        return ret;
+}
+
+int32_t
+glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo)
+{
+        rpc_clnt_t              *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
+
+        GF_ASSERT (brickinfo);
+
+        if (!brickinfo) {
+                gf_log_callingfn ("glusterd", GF_LOG_WARNING, "!brickinfo");
+                return -1;
+        }
+
+        rpc            = brickinfo->rpc;
+        brickinfo->rpc = NULL;
+
+        if (rpc) {
+                synclock_unlock (&priv->big_lock);
+                rpc_clnt_unref (rpc);
+                synclock_lock (&priv->big_lock);
+        }
+
+        return 0;
+}
+
+int32_t
+glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
+                                glusterd_brickinfo_t   *brickinfo,
+                                gf_boolean_t del_brick)
+{
+        xlator_t        *this                   = NULL;
+        glusterd_conf_t *priv                   = NULL;
+        char            pidfile[PATH_MAX]       = {0,};
+        int             ret                     = 0;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        if (del_brick)
+                list_del_init (&brickinfo->brick_list);
+
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                (void) glusterd_brick_disconnect (brickinfo);
+                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
+                ret = glusterd_service_stop ("brick", pidfile, SIGTERM, _gf_false);
+                if (ret == 0) {
+                        glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
+                        (void) glusterd_brick_unlink_socket_file (volinfo, brickinfo);
+                }
+        }
+
+        if (del_brick)
+                glusterd_delete_brick (volinfo, brickinfo);
+
+        return ret;
+}
+
+int32_t
+glusterd_peer_hostname_new (char *hostname, glusterd_peer_hostname_t **name)
+{
+        glusterd_peer_hostname_t        *peer_hostname = NULL;
+        int32_t                         ret = -1;
+
+        GF_ASSERT (hostname);
+        GF_ASSERT (name);
+
+        peer_hostname = GF_CALLOC (1, sizeof (*peer_hostname),
+                                   gf_gld_mt_peer_hostname_t);
+
+        if (!peer_hostname)
+                goto out;
+
+        peer_hostname->hostname = gf_strdup (hostname);
+        INIT_LIST_HEAD (&peer_hostname->hostname_list);
+
+        *name = peer_hostname;
+        ret = 0;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Free LINE[0..N-1] and then the LINE buffer.  */
+static void
+free_lines (char **line, size_t n)
+{
+  size_t i;
+  for (i = 0; i < n; i++)
+    GF_FREE (line[i]);
+  GF_FREE (line);
+}
+
+char **
+glusterd_readin_file (const char *filepath, int *line_count)
+{
+        int         ret                    = -1;
+        int         n                      = 8;
+        int         counter                = 0;
+        char        buffer[PATH_MAX + 256] = {0};
+        char      **lines                  = NULL;
+        FILE       *fp                     = NULL;
+        void       *p;
+
+        fp = fopen (filepath, "r");
+        if (!fp)
+                goto out;
+
+        lines = GF_CALLOC (1, n * sizeof (*lines), gf_gld_mt_charptr);
+        if (!lines)
+                goto out;
+
+        for (counter = 0; fgets (buffer, sizeof (buffer), fp); counter++) {
+
+                if (counter == n-1) {
+                        n *= 2;
+                        p = GF_REALLOC (lines, n * sizeof (char *));
+                        if (!p) {
+                                free_lines (lines, n/2);
+                                lines = NULL;
+                                goto out;
+                        }
+                        lines = p;
+                }
+
+                lines[counter] = gf_strdup (buffer);
+        }
+
+        lines[counter] = NULL;
+        /* Reduce allocation to minimal size.  */
+        p = GF_REALLOC (lines, (counter + 1) * sizeof (char *));
+        if (!p) {
+                free_lines (lines, counter);
+                lines = NULL;
+                goto out;
+        }
+        lines = p;
+
+        *line_count = counter;
+        ret = 0;
+
+ out:
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", strerror (errno));
+        if (fp)
+                fclose (fp);
+
+        return lines;
+}
+
+int
+glusterd_compare_lines (const void *a, const void *b) {
+
+        return strcmp(* (char * const *) a, * (char * const *) b);
+}
+
+int
+glusterd_sort_and_redirect (const char *src_filepath, int dest_fd)
+{
+        int            ret          = -1;
+        int            line_count   = 0;
+        int            counter      = 0;
+        char         **lines        = NULL;
+
+
+        if (!src_filepath || dest_fd < 0)
+                goto out;
+
+        lines = glusterd_readin_file (src_filepath, &line_count);
+        if (!lines)
+                goto out;
+
+        qsort (lines, line_count, sizeof (*lines), glusterd_compare_lines);
+
+        for (counter = 0; lines[counter]; counter++) {
+
+                ret = write (dest_fd, lines[counter],
+                             strlen (lines[counter]));
+                if (ret < 0)
+                        goto out;
+
+                GF_FREE (lines[counter]);
+        }
+
+        ret = 0;
+ out:
+        GF_FREE (lines);
+
+        return ret;
+}
+
+int
+glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo)
+{
+        int32_t                 ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        char                    path[PATH_MAX] = {0,};
+        char                    cksum_path[PATH_MAX] = {0,};
+        char                    filepath[PATH_MAX] = {0,};
+        int                     fd = -1;
+        uint32_t                cksum = 0;
+        char                    buf[4096] = {0,};
+        char                    sort_filepath[PATH_MAX] = {0};
+        gf_boolean_t            unlink_sortfile = _gf_false;
+        int                     sort_fd = 0;
+        xlator_t               *this = NULL;
+
+        GF_ASSERT (volinfo);
+        this = THIS;
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+
+        snprintf (cksum_path, sizeof (cksum_path), "%s/%s",
+                  path, GLUSTERD_CKSUM_FILE);
+
+        fd = open (cksum_path, O_RDWR | O_APPEND | O_CREAT| O_TRUNC, 0600);
+
+        if (-1 == fd) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to open %s, errno: %d",
+                        cksum_path, errno);
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (filepath, sizeof (filepath), "%s/%s", path,
+                  GLUSTERD_VOLUME_INFO_FILE);
+        snprintf (sort_filepath, sizeof (sort_filepath),
+                  "/tmp/%s.XXXXXX", volinfo->volname);
+
+        sort_fd = mkstemp (sort_filepath);
+        if (sort_fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not generate temp "
+                        "file, reason: %s for %s: %s", strerror (errno),
+                        (volinfo->is_snap_volume)?"snap":"volume",
+                        volinfo->volname);
+                goto out;
+        } else {
+                unlink_sortfile = _gf_true;
+        }
+
+        /* sort the info file, result in sort_filepath */
+
+        ret = glusterd_sort_and_redirect (filepath, sort_fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "sorting info file failed");
+                goto out;
+        }
+
+        ret = close (sort_fd);
+        if (ret)
+                goto out;
+
+        ret = get_checksum_for_path (sort_filepath, &cksum);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get checksum"
+                        " for path: %s", sort_filepath);
+                goto out;
+        }
+
+        snprintf (buf, sizeof (buf), "%s=%u\n", "info", cksum);
+        ret = write (fd, buf, strlen (buf));
+
+        if (ret <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = get_checksum_for_file (fd, &cksum);
+
+        if (ret)
+                goto out;
+
+        volinfo->cksum = cksum;
+out:
+        if (fd > 0)
+               close (fd);
+        if (unlink_sortfile)
+               unlink (sort_filepath);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+int
+_add_dict_to_prdict (dict_t *this, char *key, data_t *value, void *data)
+{
+        glusterd_dict_ctx_t     *ctx = NULL;
+        char                    optkey[512] = {0,};
+        int                     ret = -1;
+
+        ctx = data;
+        snprintf (optkey, sizeof (optkey), "%s.%s%d", ctx->prefix,
+                  ctx->key_name, ctx->opt_count);
+        ret = dict_set_str (ctx->dict, optkey, key);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "option add for %s%d %s",
+                        ctx->key_name, ctx->opt_count, key);
+        snprintf (optkey, sizeof (optkey), "%s.%s%d", ctx->prefix,
+                  ctx->val_name, ctx->opt_count);
+        ret = dict_set_str (ctx->dict, optkey, value->data);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "option add for %s%d %s",
+                        ctx->val_name, ctx->opt_count, value->data);
+        ctx->opt_count++;
+
+        return ret;
+}
+
+int32_t
+glusterd_add_bricks_hname_path_to_dict (dict_t *dict,
+                                        glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        int                     ret = 0;
+        char                    key[256] = {0};
+        int                     index = 0;
+
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                snprintf (key, sizeof (key), "%d-hostname", index);
+                ret = dict_set_str (dict, key, brickinfo->hostname);
+                if (ret)
+                        goto out;
+
+                snprintf (key, sizeof (key), "%d-path", index);
+                ret = dict_set_str (dict, key, brickinfo->path);
+                if (ret)
+                        goto out;
+
+                index++;
+        }
+out:
+        return ret;
+}
+
+int32_t
+glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
+                             dict_t  *dict, int32_t count)
+{
+        int32_t                 ret               = -1;
+        char                    prefix[512]       = {0,};
+        char                    key[512]          = {0,};
+        glusterd_brickinfo_t    *brickinfo        = NULL;
+        int32_t                 i                 = 1;
+        char                    *volume_id_str    = NULL;
+        char                    *src_brick        = NULL;
+        char                    *dst_brick        = NULL;
+        char                    *str              = NULL;
+        glusterd_dict_ctx_t     ctx               = {0};
+        char                    *rebalance_id_str = NULL;
+        char                    *rb_id_str        = NULL;
+        xlator_t                *this             = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (dict);
+        GF_ASSERT (volinfo);
+
+        snprintf (key, sizeof (key), "volume%d.name", count);
+        ret = dict_set_str (dict, key, volinfo->volname);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.type", count);
+        ret = dict_set_int32 (dict, key, volinfo->type);
+        if (ret)
+                goto out;
+
+        snprintf (key, sizeof (key), "volume%d.is_volume_restored", count);
+        ret = dict_set_int32 (dict, key, volinfo->is_volume_restored);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set "
+                        "is_volume_restored option for %s volume",
+                        volinfo->volname);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.brick_count", count);
+        ret = dict_set_int32 (dict, key, volinfo->brick_count);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.version", count);
+        ret = dict_set_int32 (dict, key, volinfo->version);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.status", count);
+        ret = dict_set_int32 (dict, key, volinfo->status);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.sub_count", count);
+        ret = dict_set_int32 (dict, key, volinfo->sub_count);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.stripe_count", count);
+        ret = dict_set_int32 (dict, key, volinfo->stripe_count);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.replica_count", count);
+        ret = dict_set_int32 (dict, key, volinfo->replica_count);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.dist_count", count);
+        ret = dict_set_int32 (dict, key, volinfo->dist_leaf_count);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.ckusm", count);
+        ret = dict_set_int64 (dict, key, volinfo->cksum);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.transport_type", count);
+        ret = dict_set_uint32 (dict, key, volinfo->transport_type);
+        if (ret)
+                goto out;
+
+        snprintf (key, sizeof (key), "volume%d.is_snap_volume", count);
+        ret = dict_set_uint32 (dict, key, volinfo->is_snap_volume);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to set %s", key);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.snap-max-hard-limit", count);
+        ret = dict_set_uint64 (dict, key, volinfo->snap_max_hard_limit);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to set %s", key);
+                goto out;
+        }
+
+        volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
+        if (!volume_id_str) {
+                ret = -1;
+                goto out;
+        }
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.volume_id", count);
+        ret = dict_set_dynstr (dict, key, volume_id_str);
+        if (ret)
+                goto out;
+        volume_id_str = NULL;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.username", count);
+        str = glusterd_auth_get_username (volinfo);
+        if (str) {
+                ret = dict_set_dynstr (dict, key, gf_strdup (str));
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.password", count);
+        str = glusterd_auth_get_password (volinfo);
+        if (str) {
+                ret = dict_set_dynstr (dict, key, gf_strdup (str));
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d.rebalance", count);
+        ret = dict_set_int32 (dict, key, volinfo->rebal.defrag_cmd);
+        if (ret)
+                goto out;
+
+        if (volinfo->rebal.defrag_cmd) {
+                rebalance_id_str = gf_strdup (uuid_utoa
+                                                (volinfo->rebal.rebalance_id));
+                if (!rebalance_id_str) {
+                        ret = -1;
+                        goto out;
+                }
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d.rebalance-id", count);
+                ret = dict_set_dynstr (dict, key, rebalance_id_str);
+                if (ret)
+                        goto out;
+                rebalance_id_str = NULL;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebalance-op", count);
+        ret = dict_set_uint32 (dict, key, volinfo->rebal.op);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
+        ret = dict_set_int32 (dict, key, volinfo->rep_brick.rb_status);
+        if (ret)
+                goto out;
+
+        if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                          count);
+                gf_asprintf (&src_brick, "%s:%s",
+                             volinfo->rep_brick.src_brick->hostname,
+                             volinfo->rep_brick.src_brick->path);
+                ret = dict_set_dynstr (dict, key, src_brick);
+                if (ret)
+                        goto out;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                          count);
+                gf_asprintf (&dst_brick, "%s:%s",
+                             volinfo->rep_brick.dst_brick->hostname,
+                             volinfo->rep_brick.dst_brick->path);
+                ret = dict_set_dynstr (dict, key, dst_brick);
+                if (ret)
+                        goto out;
+
+                rb_id_str = gf_strdup (uuid_utoa (volinfo->rep_brick.rb_id));
+                if (!rb_id_str) {
+                        ret = -1;
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rb_id", count);
+                ret = dict_set_dynstr (dict, key, rb_id_str);
+                if (ret)
+                        goto out;
+                rb_id_str = NULL;
+        }
+
+        snprintf (prefix, sizeof (prefix), "volume%d", count);
+        ctx.dict = dict;
+        ctx.prefix = prefix;
+        ctx.opt_count = 1;
+        ctx.key_name = "key";
+        ctx.val_name = "value";
+        GF_ASSERT (volinfo->dict);
+
+        dict_foreach (volinfo->dict, _add_dict_to_prdict, &ctx);
+        ctx.opt_count--;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.opt-count", count);
+        ret = dict_set_int32 (dict, key, ctx.opt_count);
+        if (ret)
+                goto out;
+
+        ctx.dict = dict;
+        ctx.prefix = prefix;
+        ctx.opt_count = 1;
+        ctx.key_name = "slave-num";
+        ctx.val_name = "slave-val";
+        GF_ASSERT (volinfo->gsync_slaves);
+
+        dict_foreach (volinfo->gsync_slaves, _add_dict_to_prdict, &ctx);
+        ctx.opt_count--;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.gsync-count", count);
+        ret = dict_set_int32 (dict, key, ctx.opt_count);
+        if (ret)
+                goto out;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.brick%d.hostname",
+                          count, i);
+                ret = dict_set_str (dict, key, brickinfo->hostname);
+                if (ret)
+                        goto out;
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.brick%d.path",
+                          count, i);
+                ret = dict_set_str (dict, key, brickinfo->path);
+                if (ret)
+                        goto out;
+
+                snprintf (key, sizeof (key), "volume%d.brick%d.snap_status",
+                          count, i);
+                ret = dict_set_int32 (dict, key, brickinfo->snap_status);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set snap_status for %s:%s",
+                                brickinfo->hostname,
+                                brickinfo->path);
+                        goto out;
+                }
+
+                snprintf (key, sizeof (key), "volume%d.brick%d.device_path",
+                          count, i);
+                ret = dict_set_str (dict, key, brickinfo->device_path);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set snap_device for %s:%s",
+                                brickinfo->hostname,
+                                brickinfo->path);
+                        goto out;
+                }
+
+                i++;
+        }
+
+        /* Add volume op-versions to dict. This prevents volume inconsistencies
+         * in the cluster
+         */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.op-version", count);
+        ret = dict_set_int32 (dict, key, volinfo->op_version);
+        if (ret)
+                goto out;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.client-op-version", count);
+        ret = dict_set_int32 (dict, key, volinfo->client_op_version);
+
+out:
+        GF_FREE (volume_id_str);
+        GF_FREE (rebalance_id_str);
+        GF_FREE (rb_id_str);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_build_volume_dict (dict_t **vols)
+{
+        int32_t                 ret = -1;
+        dict_t                  *dict = NULL;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        int32_t                 count = 0;
+        glusterd_dict_ctx_t     ctx            = {0};
+
+        priv = THIS->private;
+
+        dict = dict_new ();
+
+        if (!dict)
+                goto out;
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                count++;
+                ret = glusterd_add_volume_to_dict (volinfo, dict, count);
+                if (ret)
+                        goto out;
+        }
+
+
+        ret = dict_set_int32 (dict, "count", count);
+        if (ret)
+                goto out;
+
+        ctx.dict = dict;
+        ctx.prefix = "global";
+        ctx.opt_count = 1;
+        ctx.key_name = "key";
+        ctx.val_name = "val";
+        dict_foreach (priv->opts, _add_dict_to_prdict, &ctx);
+        ctx.opt_count--;
+        ret = dict_set_int32 (dict, "global-opt-count", ctx.opt_count);
+        if (ret)
+                goto out;
+
+        *vols = dict;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        if (ret)
+                dict_unref (dict);
+
+        return ret;
+}
+
+int32_t
+glusterd_compare_friend_volume (dict_t *vols, int32_t count, int32_t *status,
+                                char *hostname)
+{
+
+        int32_t                 ret = -1;
+        char                    key[512] = {0,};
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *volname = NULL;
+        uint32_t                cksum = 0;
+        int32_t                 version = 0;
+
+        GF_ASSERT (vols);
+        GF_ASSERT (status);
+
+        snprintf (key, sizeof (key), "volume%d.name", count);
+        ret = dict_get_str (vols, key, &volname);
+        if (ret)
+                goto out;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret) {
+                *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
+                ret = 0;
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.version", count);
+        ret = dict_get_int32 (vols, key, &version);
+        if (ret)
+                goto out;
+
+        if (version > volinfo->version) {
+                //Mismatch detected
+                ret = 0;
+                gf_log ("", GF_LOG_ERROR, "Version of volume %s differ."
+                        "local version = %d, remote version = %d on peer %s",
+                        volinfo->volname, volinfo->version, version, hostname);
+                *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
+                goto out;
+        } else if (version < volinfo->version) {
+		*status = GLUSTERD_VOL_COMP_SCS;
+		goto out;
+	}
+
+        //Now, versions are same, compare cksums.
+        //
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.ckusm", count);
+        ret = dict_get_uint32 (vols, key, &cksum);
+        if (ret)
+                goto out;
+
+        if (cksum != volinfo->cksum) {
+                ret = 0;
+                gf_log ("", GF_LOG_ERROR, "Cksums of volume %s differ."
+                        " local cksum = %u, remote cksum = %u on peer %s",
+                        volinfo->volname, volinfo->cksum, cksum, hostname);
+                *status = GLUSTERD_VOL_COMP_RJT;
+                goto out;
+        }
+
+        *status = GLUSTERD_VOL_COMP_SCS;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with ret: %d, status: %d",
+                ret, *status);
+        return ret;
+}
+
+static int32_t
+import_prdict_dict (dict_t *vols, dict_t  *dst_dict, char *key_prefix,
+                    char *value_prefix, int opt_count, char *prefix)
+{
+        char                    key[512] = {0,};
+        int32_t                 ret = 0;
+        int                     i = 1;
+        char                    *opt_key = NULL;
+        char                    *opt_val = NULL;
+        char                    *dup_opt_val = NULL;
+        char                    msg[2048] = {0};
+
+        while (i <= opt_count) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.%s%d",
+                          prefix, key_prefix, i);
+                ret = dict_get_str (vols, key, &opt_key);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), "Volume dict key not "
+                                  "specified");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.%s%d",
+                          prefix, value_prefix, i);
+                ret = dict_get_str (vols, key, &opt_val);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), "Volume dict value not "
+                                  "specified");
+                        goto out;
+                }
+                dup_opt_val = gf_strdup (opt_val);
+                if (!dup_opt_val) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_dynstr (dst_dict, opt_key, dup_opt_val);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), "Volume set %s %s "
+                                  "unsuccessful", opt_key, dup_opt_val);
+                        goto out;
+                }
+                i++;
+        }
+
+out:
+        if (msg[0])
+                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+
+}
+
+gf_boolean_t
+glusterd_is_quorum_option (char *option)
+{
+        gf_boolean_t    res = _gf_false;
+        int             i = 0;
+        char            *keys[] = {GLUSTERD_QUORUM_TYPE_KEY,
+                                   GLUSTERD_QUORUM_RATIO_KEY, NULL};
+
+        for (i = 0; keys[i]; i++) {
+                if (strcmp (option, keys[i]) == 0) {
+                        res = _gf_true;
+                        break;
+                }
+        }
+        return res;
+}
+
+gf_boolean_t
+glusterd_is_quorum_changed (dict_t *options, char *option, char *value)
+{
+        int             ret = 0;
+        gf_boolean_t    reconfigured = _gf_false;
+        gf_boolean_t    all = _gf_false;
+        char            *oldquorum = NULL;
+        char            *newquorum = NULL;
+        char            *oldratio = NULL;
+        char            *newratio = NULL;
+
+        if ((strcmp ("all", option) != 0) &&
+            !glusterd_is_quorum_option (option))
+                goto out;
+
+        if (strcmp ("all", option) == 0)
+                all = _gf_true;
+
+        if (all || (strcmp (GLUSTERD_QUORUM_TYPE_KEY, option) == 0)) {
+                newquorum = value;
+                ret = dict_get_str (options, GLUSTERD_QUORUM_TYPE_KEY,
+                                    &oldquorum);
+        }
+
+        if (all || (strcmp (GLUSTERD_QUORUM_RATIO_KEY, option) == 0)) {
+                newratio = value;
+                ret = dict_get_str (options, GLUSTERD_QUORUM_RATIO_KEY,
+                                    &oldratio);
+        }
+
+        reconfigured = _gf_true;
+
+        if (oldquorum && newquorum && (strcmp (oldquorum, newquorum) == 0))
+                reconfigured = _gf_false;
+        if (oldratio && newratio && (strcmp (oldratio, newratio) == 0))
+                reconfigured = _gf_false;
+
+        if ((oldratio == NULL) && (newratio == NULL) && (oldquorum == NULL) &&
+            (newquorum == NULL))
+                reconfigured = _gf_false;
+out:
+        return reconfigured;
+}
+
+static inline gf_boolean_t
+_is_contributing_to_quorum (gd_quorum_contrib_t contrib)
+{
+        if ((contrib == QUORUM_UP) || (contrib == QUORUM_DOWN))
+                return _gf_true;
+        return _gf_false;
+}
+
+static inline gf_boolean_t
+_does_quorum_meet (int active_count, int quorum_count)
+{
+        return (active_count >= quorum_count);
+}
+
+int
+glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
+                                    int *quorum_count)
+{
+        glusterd_peerinfo_t *peerinfo      = NULL;
+        glusterd_conf_t     *conf          = NULL;
+        int                 ret            = -1;
+        int                 inquorum_count = 0;
+        char                *val           = NULL;
+        double              quorum_percentage = 0.0;
+        gf_boolean_t        ratio          = _gf_false;
+        int                 count          = 0;
+
+        conf = this->private;
+        //Start with counting self
+        inquorum_count = 1;
+        if (active_count)
+                *active_count = 1;
+        list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
+                if (peerinfo->quorum_contrib == QUORUM_WAITING)
+                        goto out;
+
+                if (_is_contributing_to_quorum (peerinfo->quorum_contrib))
+                        inquorum_count = inquorum_count + 1;
+
+                if (active_count && (peerinfo->quorum_contrib == QUORUM_UP))
+                        *active_count = *active_count + 1;
+        }
+
+        ret = dict_get_str (conf->opts, GLUSTERD_QUORUM_RATIO_KEY, &val);
+        if (ret == 0) {
+                ratio = _gf_true;
+                ret = gf_string2percent (val, &quorum_percentage);
+                if (!ret)
+                        ratio = _gf_true;
+        }
+        if (ratio)
+                count = CEILING_POS (inquorum_count *
+                                     quorum_percentage / 100.0);
+        else
+                count = (inquorum_count * 50 / 100) + 1;
+
+        *quorum_count = count;
+        ret = 0;
+out:
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo)
+{
+        gf_boolean_t    res = _gf_false;
+        char            *quorum_type = NULL;
+        int             ret = 0;
+
+        ret = dict_get_str (volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY,
+                            &quorum_type);
+        if (ret)
+                goto out;
+
+        if (strcmp (quorum_type, GLUSTERD_SERVER_QUORUM) == 0)
+                res = _gf_true;
+out:
+        return res;
+}
+
+gf_boolean_t
+glusterd_is_any_volume_in_server_quorum (xlator_t *this)
+{
+        glusterd_conf_t         *conf = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+
+        conf = this->private;
+        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                if (glusterd_is_volume_in_server_quorum (volinfo)) {
+                        return _gf_true;
+                }
+        }
+        return _gf_false;
+}
+
+gf_boolean_t
+does_gd_meet_server_quorum (xlator_t *this)
+{
+        int                     quorum_count = 0;
+        int                     active_count   = 0;
+        gf_boolean_t            in = _gf_false;
+        glusterd_conf_t         *conf = NULL;
+        int                     ret = -1;
+
+        conf = this->private;
+        ret = glusterd_get_quorum_cluster_counts (this, &active_count,
+                                                  &quorum_count);
+        if (ret)
+                goto out;
+
+        if (!_does_quorum_meet (active_count, quorum_count)) {
+                goto out;
+        }
+
+        in = _gf_true;
+out:
+        return in;
+}
+
+int
+glusterd_spawn_daemons (void *opaque)
+{
+        glusterd_conf_t *conf = THIS->private;
+        gf_boolean_t    start_bricks = !conf->restart_done;
+
+        if (start_bricks) {
+                glusterd_restart_bricks (conf);
+                conf->restart_done = _gf_true;
+        }
+        glusterd_restart_gsyncds (conf);
+        glusterd_restart_rebalance (conf);
+        return 0;
+}
+
+void
+glusterd_do_volume_quorum_action (xlator_t *this, glusterd_volinfo_t *volinfo,
+                                  gf_boolean_t meets_quorum)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_conf_t         *conf      = NULL;
+
+        conf = this->private;
+        if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                goto out;
+
+        if (!glusterd_is_volume_in_server_quorum (volinfo))
+                meets_quorum = _gf_true;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (!glusterd_is_local_brick (this, volinfo, brickinfo))
+                        continue;
+                if (meets_quorum)
+                        glusterd_brick_start (volinfo, brickinfo, _gf_false);
+                else
+                        glusterd_brick_stop (volinfo, brickinfo, _gf_false);
+        }
+out:
+        return;
+}
+
+int
+glusterd_do_quorum_action ()
+{
+        xlator_t            *this          = NULL;
+        glusterd_conf_t     *conf          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                 ret            = 0;
+        int                 active_count   = 0;
+        int                 quorum_count   = 0;
+        gf_boolean_t        meets          = _gf_false;
+
+        this = THIS;
+        conf = this->private;
+
+        conf->pending_quorum_action = _gf_true;
+        ret = glusterd_lock (conf->uuid);
+        if (ret)
+                goto out;
+
+        {
+                ret = glusterd_get_quorum_cluster_counts (this, &active_count,
+                                                          &quorum_count);
+                if (ret)
+                        goto unlock;
+
+                if (_does_quorum_meet (active_count, quorum_count))
+                        meets = _gf_true;
+                list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                        glusterd_do_volume_quorum_action (this, volinfo, meets);
+                }
+        }
+unlock:
+        (void)glusterd_unlock (conf->uuid);
+        conf->pending_quorum_action = _gf_false;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_import_friend_volume_opts (dict_t *vols, int count,
+                                    glusterd_volinfo_t *volinfo)
+{
+        char                    key[512] = {0,};
+        int32_t                 ret = -1;
+        int                     opt_count = 0;
+        char                    msg[2048] = {0};
+        char                    volume_prefix[1024] = {0};
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.opt-count", count);
+        ret = dict_get_int32 (vols, key, &opt_count);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume option count not "
+                          "specified for %s", volinfo->volname);
+                goto out;
+        }
+
+        snprintf (volume_prefix, sizeof (volume_prefix), "volume%d", count);
+        ret = import_prdict_dict (vols, volinfo->dict, "key", "value",
+                                  opt_count, volume_prefix);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Unable to import options dict "
+                          "specified for %s", volinfo->volname);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.gsync-count", count);
+        ret = dict_get_int32 (vols, key, &opt_count);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Gsync count not "
+                          "specified for %s", volinfo->volname);
+                goto out;
+        }
+
+        ret = import_prdict_dict (vols, volinfo->gsync_slaves, "slave-num",
+                                  "slave-val", opt_count, volume_prefix);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Unable to import gsync sessions "
+                          "specified for %s", volinfo->volname);
+                goto out;
+        }
+
+out:
+        if (msg[0])
+                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_import_new_brick (dict_t *vols, int32_t vol_count,
+                           int32_t brick_count,
+                           glusterd_brickinfo_t **brickinfo)
+{
+        char                    key[512] = {0,};
+        int                     ret = -1;
+        int32_t                 snap_status = 0;
+        char                    *snap_device = NULL;
+        char                    *hostname = NULL;
+        char                    *path = NULL;
+        glusterd_brickinfo_t    *new_brickinfo = NULL;
+        char                    msg[2048] = {0};
+
+        GF_ASSERT (vols);
+        GF_ASSERT (vol_count >= 0);
+        GF_ASSERT (brickinfo);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.brick%d.hostname",
+                  vol_count, brick_count);
+        ret = dict_get_str (vols, key, &hostname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload", key);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.brick%d.path",
+                  vol_count, brick_count);
+        ret = dict_get_str (vols, key, &path);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload", key);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.brick%d.snap_status",
+                  vol_count, brick_count);
+        ret = dict_get_int32 (vols, key, &snap_status);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload", key);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.brick%d.device_path",
+                  vol_count, brick_count);
+        ret = dict_get_str (vols, key, &snap_device);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload", key);
+                goto out;
+        }
+
+        ret = glusterd_brickinfo_new (&new_brickinfo);
+        if (ret)
+                goto out;
+
+        strcpy (new_brickinfo->path, path);
+        strcpy (new_brickinfo->hostname, hostname);
+        strcpy (new_brickinfo->device_path, snap_device);
+        new_brickinfo->snap_status = snap_status;
+
+        //peerinfo might not be added yet
+        (void) glusterd_resolve_brick (new_brickinfo);
+        ret = 0;
+        *brickinfo = new_brickinfo;
+out:
+        if (msg[0])
+                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_import_bricks (dict_t *vols, int32_t vol_count,
+                        glusterd_volinfo_t *new_volinfo)
+{
+        int                     ret = -1;
+        int                     brick_count = 1;
+        glusterd_brickinfo_t     *new_brickinfo = NULL;
+
+        GF_ASSERT (vols);
+        GF_ASSERT (vol_count >= 0);
+        GF_ASSERT (new_volinfo);
+        while (brick_count <= new_volinfo->brick_count) {
+
+                ret = glusterd_import_new_brick (vols, vol_count, brick_count,
+                                                 &new_brickinfo);
+                if (ret)
+                        goto out;
+                list_add_tail (&new_brickinfo->brick_list, &new_volinfo->bricks);
+                brick_count++;
+        }
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_import_volinfo (dict_t *vols, int count,
+                         glusterd_volinfo_t **volinfo)
+{
+        int                ret               = -1;
+        char               key[256]          = {0};
+        char               *volname          = NULL;
+        glusterd_volinfo_t *new_volinfo      = NULL;
+        char               *volume_id_str    = NULL;
+        char               msg[2048]         = {0};
+        char               *src_brick        = NULL;
+        char               *dst_brick        = NULL;
+        char               *str              = NULL;
+        int                rb_status         = 0;
+        char               *rebalance_id_str = NULL;
+        char               *rb_id_str        = NULL;
+        int                op_version        = 0;
+        int                client_op_version = 0;
+        uint32_t           is_snap_volume    = 0;
+
+        GF_ASSERT (vols);
+        GF_ASSERT (volinfo);
+
+        snprintf (key, sizeof (key), "volume%d.name", count);
+        ret = dict_get_str (vols, key, &volname);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload", key);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.is_snap_volume", count);
+        ret = dict_get_uint32 (vols, key, &is_snap_volume);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        if (is_snap_volume == _gf_true) {
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "Not syncing snap volume %s", volname);
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_volinfo_new (&new_volinfo);
+        if (ret)
+                goto out;
+        strncpy (new_volinfo->volname, volname, sizeof (new_volinfo->volname));
+
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.type", count);
+        ret = dict_get_int32 (vols, key, &new_volinfo->type);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.brick_count", count);
+        ret = dict_get_int32 (vols, key, &new_volinfo->brick_count);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.version", count);
+        ret = dict_get_int32 (vols, key, &new_volinfo->version);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.status", count);
+        ret = dict_get_int32 (vols, key, (int32_t *)&new_volinfo->status);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.sub_count", count);
+        ret = dict_get_int32 (vols, key, &new_volinfo->sub_count);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        /* not having a 'stripe_count' key is not a error
+           (as peer may be of old version) */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.stripe_count", count);
+        ret = dict_get_int32 (vols, key, &new_volinfo->stripe_count);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_INFO,
+                        "peer is possibly old version");
+
+        /* not having a 'replica_count' key is not a error
+           (as peer may be of old version) */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.replica_count", count);
+        ret = dict_get_int32 (vols, key, &new_volinfo->replica_count);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_INFO,
+                        "peer is possibly old version");
+
+        /* not having a 'dist_count' key is not a error
+           (as peer may be of old version) */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.dist_count", count);
+        ret = dict_get_int32 (vols, key, &new_volinfo->dist_leaf_count);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_INFO,
+                        "peer is possibly old version");
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.ckusm", count);
+        ret = dict_get_uint32 (vols, key, &new_volinfo->cksum);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.volume_id", count);
+        ret = dict_get_str (vols, key, &volume_id_str);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        uuid_parse (volume_id_str, new_volinfo->volume_id);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.username", count);
+        ret = dict_get_str (vols, key, &str);
+        if (!ret) {
+                ret = glusterd_auth_set_username (new_volinfo, str);
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.password", count);
+        ret = dict_get_str (vols, key, &str);
+        if (!ret) {
+                ret = glusterd_auth_set_password (new_volinfo, str);
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.transport_type", count);
+        ret = dict_get_uint32 (vols, key, &new_volinfo->transport_type);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        new_volinfo->is_snap_volume = is_snap_volume;
+
+        snprintf (key, sizeof (key), "volume%d.is_volume_restored", count);
+        ret = dict_get_uint32 (vols, key, &new_volinfo->is_volume_restored);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get "
+                        "is_volume_restored option for %s",
+                        volname);
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "volume%d.snap-max-hard-limit", count);
+        ret = dict_get_uint64 (vols, key, &new_volinfo->snap_max_hard_limit);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebalance", count);
+        ret = dict_get_uint32 (vols, key, &new_volinfo->rebal.defrag_cmd);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
+                          key, volname);
+                goto out;
+        }
+
+        if (new_volinfo->rebal.defrag_cmd) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rebalance-id", count);
+                ret = dict_get_str (vols, key, &rebalance_id_str);
+                if (ret) {
+                        /* This is not present in older glusterfs versions,
+                         * so don't error out
+                         */
+                        ret = 0;
+                } else {
+                        uuid_parse (rebalance_id_str,
+                                    new_volinfo->rebal.rebalance_id);
+                }
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.rebalance-op", count);
+        ret = dict_get_uint32 (vols, key,(uint32_t *) &new_volinfo->rebal.op);
+        if (ret) {
+                /* This is not present in older glusterfs versions,
+                 * so don't error out
+                 */
+                ret = 0;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_STATUS, count);
+        ret = dict_get_int32 (vols, key, &rb_status);
+        if (ret)
+                goto out;
+        new_volinfo->rep_brick.rb_status = rb_status;
+
+        if (new_volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
+                          count);
+                ret = dict_get_str (vols, key, &src_brick);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_brickinfo_new_from_brick (src_brick,
+                                        &new_volinfo->rep_brick.src_brick);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to create"
+                                " src brickinfo");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, 256, "volume%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
+                          count);
+                ret = dict_get_str (vols, key, &dst_brick);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_brickinfo_new_from_brick (dst_brick,
+                                     &new_volinfo->rep_brick.dst_brick);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to create"
+                                " dst brickinfo");
+                        goto out;
+                }
+
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "volume%d.rb_id", count);
+                ret = dict_get_str (vols, key, &rb_id_str);
+                if (ret) {
+                        /* This is not present in older glusterfs versions,
+                         * so don't error out
+                         */
+                        ret = 0;
+                } else {
+                        uuid_parse (rb_id_str, new_volinfo->rep_brick.rb_id);
+                }
+        }
+
+
+        ret = glusterd_import_friend_volume_opts (vols, count, new_volinfo);
+        if (ret)
+                goto out;
+
+        /* Import the volume's op-versions if available else set it to 1.
+         * Not having op-versions implies this informtation was obtained from a
+         * op-version 1 friend (gluster-3.3), ergo the cluster is at op-version
+         * 1 and all volumes are at op-versions 1.
+         *
+         * Either both the volume op-versions should be absent or both should be
+         * present. Only one being present is a failure
+         */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.op-version", count);
+        ret = dict_get_int32 (vols, key, &op_version);
+        if (ret)
+                ret = 0;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "volume%d.client-op-version", count);
+        ret = dict_get_int32 (vols, key, &client_op_version);
+        if (ret)
+                ret = 0;
+
+        if (op_version && client_op_version) {
+                new_volinfo->op_version = op_version;
+                new_volinfo->client_op_version = client_op_version;
+        } else if (((op_version == 0) && (client_op_version != 0)) ||
+                   ((op_version != 0) && (client_op_version == 0))) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "Only one volume op-version found");
+                goto out;
+        } else {
+                new_volinfo->op_version = 1;
+                new_volinfo->client_op_version = 1;
+        }
+
+        ret = glusterd_import_bricks (vols, count, new_volinfo);
+        if (ret)
+                goto out;
+        *volinfo = new_volinfo;
+out:
+        if (msg[0])
+                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_volume_disconnect_all_bricks (glusterd_volinfo_t *volinfo)
+{
+        int                  ret = 0;
+        glusterd_brickinfo_t *brickinfo = NULL;
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (glusterd_is_brick_started (brickinfo)) {
+                        ret = glusterd_brick_disconnect (brickinfo);
+                        if (ret) {
+                                gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                        "disconnect %s:%s", brickinfo->hostname,
+                                        brickinfo->path);
+                                break;
+                        }
+                }
+        }
+
+        return ret;
+}
+
+int32_t
+glusterd_volinfo_copy_brick_portinfo (glusterd_volinfo_t *new_volinfo,
+                                      glusterd_volinfo_t *old_volinfo)
+{
+        glusterd_brickinfo_t *new_brickinfo = NULL;
+        glusterd_brickinfo_t *old_brickinfo = NULL;
+
+        int             ret = 0;
+        GF_ASSERT (new_volinfo);
+        GF_ASSERT (old_volinfo);
+        if (_gf_false == glusterd_is_volume_started (new_volinfo))
+                goto out;
+        list_for_each_entry (new_brickinfo, &new_volinfo->bricks, brick_list) {
+                ret = glusterd_volume_brickinfo_get (new_brickinfo->uuid,
+                                                     new_brickinfo->hostname,
+                                                     new_brickinfo->path,
+                                                     old_volinfo, &old_brickinfo);
+                if ((0 == ret) && glusterd_is_brick_started (old_brickinfo)) {
+                        new_brickinfo->port = old_brickinfo->port;
+                }
+        }
+out:
+        ret = 0;
+        return ret;
+}
+
+int32_t
+glusterd_volinfo_stop_stale_bricks (glusterd_volinfo_t *new_volinfo,
+                                    glusterd_volinfo_t *old_volinfo)
+{
+        glusterd_brickinfo_t *new_brickinfo = NULL;
+        glusterd_brickinfo_t *old_brickinfo = NULL;
+
+        int             ret = 0;
+        GF_ASSERT (new_volinfo);
+        GF_ASSERT (old_volinfo);
+        if (_gf_false == glusterd_is_volume_started (old_volinfo))
+                goto out;
+        list_for_each_entry (old_brickinfo, &old_volinfo->bricks, brick_list) {
+                ret = glusterd_volume_brickinfo_get (old_brickinfo->uuid,
+                                                     old_brickinfo->hostname,
+                                                     old_brickinfo->path,
+                                                     new_volinfo, &new_brickinfo);
+                if (ret) {
+                        /*TODO: may need to switch to 'atomic' flavour of
+                         * brick_stop, once we make peer rpc program also
+                         * synctask enabled*/
+                        ret = glusterd_brick_stop (old_volinfo, old_brickinfo,
+                                                   _gf_false);
+                        if (ret)
+                                gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                        "stop brick %s:%s", old_brickinfo->hostname,
+                                        old_brickinfo->path);
+                }
+        }
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_delete_stale_volume (glusterd_volinfo_t *stale_volinfo,
+                              glusterd_volinfo_t *valid_volinfo)
+{
+        GF_ASSERT (stale_volinfo);
+        GF_ASSERT (valid_volinfo);
+
+        /* If stale volume is in started state, copy the port numbers of the
+         * local bricks if they exist in the valid volume information.
+         * stop stale bricks. Stale volume information is going to be deleted.
+         * Which deletes the valid brick information inside stale volinfo.
+         * We dont want brick_rpc_notify to access already deleted brickinfo.
+         * Disconnect all bricks from stale_volinfo (unconditionally), since
+         * they are being deleted subsequently.
+         */
+        if (glusterd_is_volume_started (stale_volinfo)) {
+                if (glusterd_is_volume_started (valid_volinfo)) {
+                        (void) glusterd_volinfo_stop_stale_bricks (valid_volinfo,
+                                                                   stale_volinfo);
+                        //Only valid bricks will be running now.
+                        (void) glusterd_volinfo_copy_brick_portinfo (valid_volinfo,
+                                                                     stale_volinfo);
+
+                } else {
+                        (void) glusterd_stop_bricks (stale_volinfo);
+                }
+
+                (void) glusterd_volume_disconnect_all_bricks (stale_volinfo);
+        }
+        /* Delete all the bricks and stores and vol files. They will be created
+         * again by the valid_volinfo. Volume store delete should not be
+         * performed because some of the bricks could still be running,
+         * keeping pid files under run directory
+         */
+        (void) glusterd_delete_all_bricks (stale_volinfo);
+        if (stale_volinfo->shandle) {
+                unlink (stale_volinfo->shandle->path);
+                (void) gf_store_handle_destroy (stale_volinfo->shandle);
+                stale_volinfo->shandle = NULL;
+        }
+        (void) glusterd_volinfo_delete (stale_volinfo);
+        return 0;
+}
+
+int32_t
+glusterd_import_friend_volume (dict_t *vols, size_t count)
+{
+
+        int32_t                 ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        xlator_t                *this = NULL;
+        glusterd_volinfo_t      *old_volinfo = NULL;
+        glusterd_volinfo_t      *new_volinfo = NULL;
+
+        GF_ASSERT (vols);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        ret = glusterd_import_volinfo (vols, count, &new_volinfo);
+        if (ret)
+                goto out;
+
+        if (!new_volinfo) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not importing snap volume");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (new_volinfo->volname, &old_volinfo);
+        if (0 == ret) {
+                (void) glusterd_delete_stale_volume (old_volinfo, new_volinfo);
+        }
+
+        if (glusterd_is_volume_started (new_volinfo)) {
+                (void) glusterd_start_bricks (new_volinfo);
+        }
+
+        ret = glusterd_store_volinfo (new_volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
+        ret = glusterd_create_volfiles_and_notify_services (new_volinfo);
+        if (ret)
+                goto out;
+
+        list_add_tail (&new_volinfo->vol_list, &priv->volumes);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with ret: %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_import_friend_volumes (dict_t  *vols)
+{
+        int32_t                 ret = -1;
+        int32_t                 count = 0;
+        int                     i = 1;
+
+        GF_ASSERT (vols);
+
+        ret = dict_get_int32 (vols, "count", &count);
+        if (ret)
+                goto out;
+
+        while (i <= count) {
+                ret = glusterd_import_friend_volume (vols, i);
+                if (ret)
+                        goto out;
+                i++;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
+glusterd_get_global_opt_version (dict_t *opts, uint32_t *version)
+{
+        int     ret = -1;
+        char    *version_str = NULL;
+
+        ret = dict_get_str (opts, GLUSTERD_GLOBAL_OPT_VERSION, &version_str);
+        if (ret)
+                goto out;
+
+        ret = gf_string2uint (version_str, version);
+        if (ret)
+                goto out;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_get_next_global_opt_version_str (dict_t *opts, char **version_str)
+{
+        int             ret = -1;
+        char            version_string[64] = {0};
+        uint32_t        version = 0;
+
+        ret = glusterd_get_global_opt_version (opts, &version);
+        if (ret)
+                goto out;
+        version++;
+        snprintf (version_string, sizeof (version_string), "%"PRIu32, version);
+        *version_str = gf_strdup (version_string);
+        if (*version_str)
+                ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+glusterd_import_global_opts (dict_t *friend_data)
+{
+        xlator_t        *this = NULL;
+        glusterd_conf_t *conf = NULL;
+        int             ret = -1;
+        dict_t          *import_options = NULL;
+        int             count = 0;
+        uint32_t        local_version = 0;
+        uint32_t        remote_version = 0;
+
+        this = THIS;
+        conf = this->private;
+
+        ret = dict_get_int32 (friend_data, "global-opt-count", &count);
+        if (ret) {
+                //old version peer
+                ret = 0;
+                goto out;
+        }
+
+        import_options = dict_new ();
+        if (!import_options)
+                goto out;
+        ret = import_prdict_dict (friend_data, import_options, "key", "val",
+                                  count, "global");
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to import"
+                        " global options");
+                goto out;
+        }
+
+        ret = glusterd_get_global_opt_version (conf->opts, &local_version);
+        if (ret)
+                goto out;
+        ret = glusterd_get_global_opt_version (import_options, &remote_version);
+        if (ret)
+                goto out;
+        if (remote_version > local_version) {
+                ret = glusterd_store_options (this, import_options);
+                if (ret)
+                        goto out;
+                dict_unref (conf->opts);
+                conf->opts = dict_ref (import_options);
+        }
+        ret = 0;
+out:
+        if (import_options)
+                dict_unref (import_options);
+        return ret;
+}
+
+int32_t
+glusterd_compare_friend_data (dict_t  *vols, int32_t *status, char *hostname)
+{
+        int32_t                 ret = -1;
+        int32_t                 count = 0;
+        int                     i = 1;
+        gf_boolean_t            update = _gf_false;
+        gf_boolean_t            stale_nfs = _gf_false;
+        gf_boolean_t            stale_shd = _gf_false;
+
+        GF_ASSERT (vols);
+        GF_ASSERT (status);
+
+        ret = dict_get_int32 (vols, "count", &count);
+        if (ret)
+                goto out;
+
+        while (i <= count) {
+                ret = glusterd_compare_friend_volume (vols, i, status,
+                                                      hostname);
+                if (ret)
+                        goto out;
+
+                if (GLUSTERD_VOL_COMP_RJT == *status) {
+                        ret = 0;
+                        goto out;
+                }
+                if (GLUSTERD_VOL_COMP_UPDATE_REQ == *status)
+                        update = _gf_true;
+
+                i++;
+        }
+
+        if (update) {
+                if (glusterd_is_nodesvc_running ("nfs"))
+                        stale_nfs = _gf_true;
+                if (glusterd_is_nodesvc_running ("glustershd"))
+                        stale_shd = _gf_true;
+                ret = glusterd_import_global_opts (vols);
+                if (ret)
+                        goto out;
+                ret = glusterd_import_friend_volumes (vols);
+                if (ret)
+                        goto out;
+                if (_gf_false == glusterd_are_all_volumes_stopped ()) {
+                        ret = glusterd_nodesvcs_handle_graph_change (NULL);
+                } else {
+                        if (stale_nfs)
+                                glusterd_nfs_server_stop ();
+                        if (stale_shd)
+                                glusterd_shd_stop ();
+                }
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning with ret: %d, status: %d",
+                ret, *status);
+
+        return ret;
+}
+
+/* Valid only in if service is 'local' to glusterd.
+ * pid can be -1, if reading pidfile failed */
+gf_boolean_t
+glusterd_is_service_running (char *pidfile, int *pid)
+{
+        FILE            *file = NULL;
+        gf_boolean_t    running = _gf_false;
+        int             ret = 0;
+        int             fno = 0;
+
+        file = fopen (pidfile, "r+");
+        if (!file)
+                goto out;
+
+        fno = fileno (file);
+        ret = lockf (fno, F_TEST, 0);
+        if (ret == -1)
+                running = _gf_true;
+        if (!pid)
+                goto out;
+
+        ret = fscanf (file, "%d", pid);
+        if (ret <= 0) {
+                gf_log ("", GF_LOG_ERROR, "Unable to read pidfile: %s, %s",
+                        pidfile, strerror (errno));
+                *pid = -1;
+        }
+
+out:
+        if (file)
+                fclose (file);
+        return running;
+}
+
+void
+glusterd_get_nodesvc_dir (char *server, char *workdir,
+                                char *path, size_t len)
+{
+        GF_ASSERT (len == PATH_MAX);
+        snprintf (path, len, "%s/%s", workdir, server);
+}
+
+void
+glusterd_get_nodesvc_rundir (char *server, char *workdir,
+                                   char *path, size_t len)
+{
+        char    dir[PATH_MAX] = {0};
+        GF_ASSERT (len == PATH_MAX);
+
+        glusterd_get_nodesvc_dir (server, workdir, dir, sizeof (dir));
+        snprintf (path, len, "%s/run", dir);
+}
+
+void
+glusterd_get_nodesvc_pidfile (char *server, char *workdir,
+                                    char *path, size_t len)
+{
+        char    dir[PATH_MAX] = {0};
+        GF_ASSERT (len == PATH_MAX);
+
+        glusterd_get_nodesvc_rundir (server, workdir, dir, sizeof (dir));
+        snprintf (path, len, "%s/%s.pid", dir, server);
+}
+
+void
+glusterd_get_nodesvc_volfile (char *server, char *workdir,
+                                    char *volfile, size_t len)
+{
+        char  dir[PATH_MAX] = {0,};
+        GF_ASSERT (len == PATH_MAX);
+
+        glusterd_get_nodesvc_dir (server, workdir, dir, sizeof (dir));
+        snprintf (volfile, len, "%s/%s-server.vol", dir, server);
+}
+
+void
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status)
+{
+        glusterd_conf_t *priv = NULL;
+
+        GF_ASSERT (server);
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (priv->shd);
+        GF_ASSERT (priv->nfs);
+
+        if (!strcmp("glustershd", server))
+                priv->shd->online = status;
+        else if (!strcmp ("nfs", server))
+                priv->nfs->online = status;
+}
+
+gf_boolean_t
+glusterd_is_nodesvc_online (char *server)
+{
+        glusterd_conf_t *conf = NULL;
+        gf_boolean_t    online = _gf_false;
+
+        GF_ASSERT (server);
+        conf = THIS->private;
+        GF_ASSERT (conf);
+        GF_ASSERT (conf->shd);
+        GF_ASSERT (conf->nfs);
+
+        if (!strcmp (server, "glustershd"))
+                online = conf->shd->online;
+        else if (!strcmp (server, "nfs"))
+                online = conf->nfs->online;
+
+        return online;
+}
+
+int32_t
+glusterd_nodesvc_set_socket_filepath (char *rundir, uuid_t uuid,
+                                      char *socketpath, int len)
+{
+        char                    sockfilepath[PATH_MAX] = {0,};
+
+        snprintf (sockfilepath, sizeof (sockfilepath), "%s/run-%s",
+                  rundir, uuid_utoa (uuid));
+
+        glusterd_set_socket_filepath (sockfilepath, socketpath, len);
+        return 0;
+}
+
+struct rpc_clnt*
+glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
+{
+        struct rpc_clnt *rpc = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        nodesrv_t               *shd       = NULL;
+        glusterd_volinfo_t      *volinfo   = NULL;
+        nodesrv_t               *nfs       = NULL;
+
+        GF_VALIDATE_OR_GOTO (THIS->name, pending_node, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, pending_node->node, out);
+
+        if (pending_node->type == GD_NODE_BRICK) {
+                brickinfo = pending_node->node;
+                rpc       = brickinfo->rpc;
+
+        } else if (pending_node->type == GD_NODE_SHD) {
+                shd       = pending_node->node;
+                rpc       = shd->rpc;
+
+        } else if (pending_node->type == GD_NODE_REBALANCE) {
+                volinfo = pending_node->node;
+                if (volinfo->rebal.defrag)
+                        rpc = volinfo->rebal.defrag->rpc;
+
+        } else if (pending_node->type == GD_NODE_NFS) {
+                nfs = pending_node->node;
+                rpc = nfs->rpc;
+
+        } else {
+                GF_ASSERT (0);
+        }
+
+out:
+        return rpc;
+}
+
+struct rpc_clnt*
+glusterd_nodesvc_get_rpc (char *server)
+{
+        glusterd_conf_t *priv   = NULL;
+        struct rpc_clnt *rpc    = NULL;
+
+        GF_ASSERT (server);
+        priv = THIS->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (priv->shd);
+        GF_ASSERT (priv->nfs);
+
+        if (!strcmp (server, "glustershd"))
+                rpc = priv->shd->rpc;
+        else if (!strcmp (server, "nfs"))
+                rpc = priv->nfs->rpc;
+
+        return rpc;
+}
+
+int32_t
+glusterd_nodesvc_set_rpc (char *server, struct rpc_clnt *rpc)
+{
+        int             ret   = 0;
+        xlator_t        *this = NULL;
+        glusterd_conf_t *priv = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+        GF_ASSERT (priv->shd);
+        GF_ASSERT (priv->nfs);
+
+        if (!strcmp ("glustershd", server))
+                priv->shd->rpc = rpc;
+        else if (!strcmp ("nfs", server))
+                priv->nfs->rpc = rpc;
+
+        return ret;
+}
+
+int32_t
+glusterd_nodesvc_connect (char *server, char *socketpath) {
+        int                     ret = 0;
+        dict_t                  *options = NULL;
+        struct rpc_clnt         *rpc = NULL;
+        glusterd_conf_t         *priv = THIS->private;
+
+        rpc = glusterd_nodesvc_get_rpc (server);
+
+        if (rpc == NULL) {
+                /* Setting frame-timeout to 10mins (600seconds).
+                 * Unix domain sockets ensures that the connection is reliable.
+                 * The default timeout of 30mins used for unreliable network
+                 * connections is too long for unix domain socket connections.
+                 */
+                ret = rpc_transport_unix_options_build (&options, socketpath,
+                                                        600);
+                if (ret)
+                        goto out;
+                synclock_unlock (&priv->big_lock);
+                ret = glusterd_rpc_create (&rpc, options,
+                                           glusterd_nodesvc_rpc_notify,
+                                           server);
+                synclock_lock (&priv->big_lock);
+                if (ret)
+                        goto out;
+                (void) glusterd_nodesvc_set_rpc (server, rpc);
+        }
+out:
+        return ret;
+}
+
+int32_t
+glusterd_nodesvc_disconnect (char *server)
+{
+        struct rpc_clnt         *rpc = NULL;
+
+        rpc = glusterd_nodesvc_get_rpc (server);
+        (void)glusterd_nodesvc_set_rpc (server, NULL);
+
+        if (rpc)
+                rpc_clnt_unref (rpc);
+
+        return 0;
+}
+
+int32_t
+glusterd_nodesvc_start (char *server)
+{
+        int32_t                 ret                        = -1;
+        xlator_t               *this                       = NULL;
+        glusterd_conf_t        *priv                       = NULL;
+        runner_t                runner                     = {0,};
+        char                    pidfile[PATH_MAX]          = {0,};
+        char                    logfile[PATH_MAX]          = {0,};
+        char                    volfile[PATH_MAX]          = {0,};
+        char                    rundir[PATH_MAX]           = {0,};
+        char                    sockfpath[PATH_MAX] = {0,};
+        char                    volfileid[256]             = {0};
+        char                    glusterd_uuid_option[1024] = {0};
+        char                    valgrind_logfile[PATH_MAX] = {0};
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        priv = this->private;
+
+        glusterd_get_nodesvc_rundir (server, priv->workdir,
+                                     rundir, sizeof (rundir));
+        ret = mkdir (rundir, 0777);
+
+        if ((ret == -1) && (EEXIST != errno)) {
+                gf_log ("", GF_LOG_ERROR, "Unable to create rundir %s",
+                        rundir);
+                goto out;
+        }
+
+        glusterd_get_nodesvc_pidfile (server, priv->workdir,
+                                      pidfile, sizeof (pidfile));
+        glusterd_get_nodesvc_volfile (server, priv->workdir,
+                                      volfile, sizeof (volfile));
+        ret = access (volfile, F_OK);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "%s Volfile %s is not present",
+                        server, volfile);
+                goto out;
+        }
+
+        snprintf (logfile, PATH_MAX, "%s/%s.log", DEFAULT_LOG_FILE_DIRECTORY,
+                  server);
+        snprintf (volfileid, sizeof (volfileid), "gluster/%s", server);
+
+        glusterd_nodesvc_set_socket_filepath (rundir, MY_UUID,
+                                              sockfpath, sizeof (sockfpath));
+
+        runinit (&runner);
+
+        if (priv->valgrind) {
+                snprintf (valgrind_logfile, PATH_MAX,
+                          "%s/valgrind-%s.log",
+                          DEFAULT_LOG_FILE_DIRECTORY,
+                          server);
+
+                runner_add_args (&runner, "valgrind", "--leak-check=full",
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
+                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+        }
+
+        runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                         "-s", "localhost",
+                         "--volfile-id", volfileid,
+                         "-p", pidfile,
+                         "-l", logfile,
+                         "-S", sockfpath, NULL);
+
+        if (!strcmp (server, "glustershd")) {
+                snprintf (glusterd_uuid_option, sizeof (glusterd_uuid_option),
+                          "*replicate*.node-uuid=%s", uuid_utoa (MY_UUID));
+                runner_add_args (&runner, "--xlator-option",
+                                 glusterd_uuid_option, NULL);
+        }
+        runner_log (&runner, "", GF_LOG_DEBUG,
+                    "Starting the nfs/glustershd services");
+
+        ret = runner_run_nowait (&runner);
+        if (ret == 0) {
+                glusterd_nodesvc_connect (server, sockfpath);
+        }
+out:
+        return ret;
+}
+
+int
+glusterd_nfs_server_start ()
+{
+        return glusterd_nodesvc_start ("nfs");
+}
+
+int
+glusterd_shd_start ()
+{
+        return glusterd_nodesvc_start ("glustershd");
+}
+
+gf_boolean_t
+glusterd_is_nodesvc_running (char *server)
+{
+        char                    pidfile[PATH_MAX] = {0,};
+        glusterd_conf_t         *priv = THIS->private;
+
+        glusterd_get_nodesvc_pidfile (server, priv->workdir,
+                                            pidfile, sizeof (pidfile));
+        return glusterd_is_service_running (pidfile, NULL);
+}
+
+int32_t
+glusterd_nodesvc_unlink_socket_file (char *server)
+{
+        int             ret = 0;
+        char            sockfpath[PATH_MAX] = {0,};
+        char            rundir[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = THIS->private;
+
+        glusterd_get_nodesvc_rundir (server, priv->workdir,
+                                     rundir, sizeof (rundir));
+
+        glusterd_nodesvc_set_socket_filepath (rundir, MY_UUID,
+                                              sockfpath, sizeof (sockfpath));
+
+        ret = unlink (sockfpath);
+        if (ret && (ENOENT == errno)) {
+                ret = 0;
+        } else {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to remove %s"
+                        " error: %s", sockfpath, strerror (errno));
+        }
+
+        return ret;
+}
+
+int32_t
+glusterd_nodesvc_stop (char *server, int sig)
+{
+        char                    pidfile[PATH_MAX] = {0,};
+        glusterd_conf_t         *priv = THIS->private;
+        int                     ret = 0;
+
+        if (!glusterd_is_nodesvc_running (server))
+                goto out;
+
+        (void)glusterd_nodesvc_disconnect (server);
+
+        glusterd_get_nodesvc_pidfile (server, priv->workdir,
+                                            pidfile, sizeof (pidfile));
+        ret = glusterd_service_stop (server, pidfile, sig, _gf_true);
+
+        if (ret == 0) {
+                glusterd_nodesvc_set_online_status (server, _gf_false);
+                (void)glusterd_nodesvc_unlink_socket_file (server);
+        }
+out:
+        return ret;
+}
+
+void
+glusterd_nfs_pmap_deregister ()
+{
+        if (pmap_unset (MOUNT_PROGRAM, MOUNTV3_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered MOUNTV3 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-register MOUNTV3 is unsuccessful");
+
+        if (pmap_unset (MOUNT_PROGRAM, MOUNTV1_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered MOUNTV1 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-register MOUNTV1 is unsuccessful");
+
+        if (pmap_unset (NFS_PROGRAM, NFSV3_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered NFSV3 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-register NFSV3 is unsuccessful");
+
+        if (pmap_unset (NLM_PROGRAM, NLMV4_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered NLM v4 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-registration of NLM v4 failed");
+
+        if (pmap_unset (NLM_PROGRAM, NLMV1_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered NLM v1 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-registration of NLM v1 failed");
+
+        if (pmap_unset (ACL_PROGRAM, ACLV3_VERSION))
+                gf_log ("", GF_LOG_INFO, "De-registered ACL v3 successfully");
+        else
+                gf_log ("", GF_LOG_ERROR, "De-registration of ACL v3 failed");
+}
+
+int
+glusterd_nfs_server_stop ()
+{
+        int                     ret = 0;
+        gf_boolean_t            deregister = _gf_false;
+
+        if (glusterd_is_nodesvc_running ("nfs"))
+                deregister = _gf_true;
+        ret = glusterd_nodesvc_stop ("nfs", SIGKILL);
+        if (ret)
+                goto out;
+        if (deregister)
+                glusterd_nfs_pmap_deregister ();
+out:
+        return ret;
+}
+
+int
+glusterd_shd_stop ()
+{
+        return glusterd_nodesvc_stop ("glustershd", SIGTERM);
+}
+
+int
+glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
+                           dict_t *vol_opts)
+{
+        int                     ret = -1;
+        glusterd_conf_t         *priv = THIS->private;
+        char                    pidfile[PATH_MAX] = {0,};
+        gf_boolean_t            running = _gf_false;
+        int                     pid = -1;
+        int                     port = 0;
+        char                    key[1024] = {0,};
+
+        glusterd_get_nodesvc_pidfile (server, priv->workdir, pidfile,
+                                      sizeof (pidfile));
+        //Consider service to be running only when glusterd sees it Online
+        if (glusterd_is_nodesvc_online (server))
+                running = glusterd_is_service_running (pidfile, &pid);
+
+        /* For nfs-servers/self-heal-daemon setting
+         * brick<n>.hostname = "NFS Server" / "Self-heal Daemon"
+         * brick<n>.path = uuid
+         * brick<n>.port = 0
+         *
+         * This might be confusing, but cli displays the name of
+         * the brick as hostname+path, so this will make more sense
+         * when output.
+         */
+        snprintf (key, sizeof (key), "brick%d.hostname", count);
+        if (!strcmp (server, "nfs"))
+                ret = dict_set_str (dict, key, "NFS Server");
+        else if (!strcmp (server, "glustershd"))
+                ret = dict_set_str (dict, key, "Self-heal Daemon");
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.path", count);
+        ret = dict_set_dynstr (dict, key, gf_strdup (uuid_utoa (MY_UUID)));
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.port", count);
+        /* Port is available only for the NFS server.
+         * Self-heal daemon doesn't provide any port for access
+         * by entities other than gluster.
+         */
+        if (!strcmp (server, "nfs")) {
+                if (dict_get (vol_opts, "nfs.port")) {
+                        ret = dict_get_int32 (vol_opts, "nfs.port", &port);
+                        if (ret)
+                                goto out;
+                } else
+                        port = GF_NFS3_PORT;
+        }
+        ret = dict_set_int32 (dict, key, port);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.pid", count);
+        ret = dict_set_int32 (dict, key, pid);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.status", count);
+        ret = dict_set_int32 (dict, key, running);
+        if (ret)
+                goto out;
+
+
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_remote_hostname_get (rpcsvc_request_t *req, char *remote_host, int len)
+{
+        GF_ASSERT (req);
+        GF_ASSERT (remote_host);
+        GF_ASSERT (req->trans);
+
+        char *name = NULL;
+        char *hostname = NULL;
+        char *tmp_host = NULL;
+        int  ret = 0;
+
+        name = req->trans->peerinfo.identifier;
+        tmp_host = gf_strdup (name);
+        if (tmp_host)
+                get_host_name (tmp_host, &hostname);
+
+        GF_ASSERT (hostname);
+        if (!hostname) {
+                memset (remote_host, 0, len);
+                ret = -1;
+                goto out;
+        }
+
+        strncpy (remote_host, hostname, strlen (hostname));
+
+
+out:
+        GF_FREE (tmp_host);
+        return ret;
+}
+
+int
+glusterd_check_generate_start_service (int (*create_volfile) (),
+                                       int (*stop) (), int (*start) ())
+{
+        int ret = -1;
+
+        ret = create_volfile ();
+        if (ret)
+                goto out;
+
+        ret = stop ();
+        if (ret)
+                goto out;
+
+        ret = start ();
+out:
+        return ret;
+}
+
+int
+glusterd_reconfigure_nodesvc (int (*create_volfile) ())
+{
+        int ret = -1;
+
+        ret = create_volfile ();
+        if (ret)
+                goto out;
+
+        ret = glusterd_fetchspec_notify (THIS);
+out:
+        return ret;
+}
+
+int
+glusterd_reconfigure_shd ()
+{
+        int (*create_volfile) () = glusterd_create_shd_volfile;
+        return glusterd_reconfigure_nodesvc (create_volfile);
+}
+
+int
+glusterd_reconfigure_nfs ()
+{
+        int             ret             = -1;
+        gf_boolean_t    identical       = _gf_false;
+
+        /*
+         * Check both OLD and NEW volfiles, if they are SAME by size
+         * and cksum i.e. "character-by-character". If YES, then
+         * NOTHING has been changed, just return.
+         */
+        ret = glusterd_check_nfs_volfile_identical (&identical);
+        if (ret)
+                goto out;
+
+        if (identical) {
+                ret = 0;
+                goto out;
+        }
+
+        /*
+         * They are not identical. Find out if the topology is changed
+         * OR just the volume options. If just the options which got
+         * changed, then inform the xlator to reconfigure the options.
+         */
+        identical = _gf_false; /* RESET the FLAG */
+        ret = glusterd_check_nfs_topology_identical (&identical);
+        if (ret)
+                goto out;
+
+        /* Topology is not changed, but just the options. But write the
+         * options to NFS volfile, so that NFS will be reconfigured.
+         */
+        if (identical) {
+                ret = glusterd_create_nfs_volfile();
+                if (ret == 0) {/* Only if above PASSES */
+                        ret = glusterd_fetchspec_notify (THIS);
+                }
+                goto out;
+        }
+
+        /*
+         * NFS volfile's topology has been changed. NFS server needs
+         * to be RESTARTED to ACT on the changed volfile.
+         */
+        ret = glusterd_check_generate_start_nfs ();
+
+out:
+        return ret;
+}
+
+
+int
+glusterd_check_generate_start_nfs ()
+{
+        int ret = 0;
+
+        ret = glusterd_check_generate_start_service (glusterd_create_nfs_volfile,
+                                                     glusterd_nfs_server_stop,
+                                                     glusterd_nfs_server_start);
+        return ret;
+}
+
+int
+glusterd_check_generate_start_shd ()
+{
+        int ret = 0;
+
+        ret = glusterd_check_generate_start_service (glusterd_create_shd_volfile,
+                                                     glusterd_shd_stop,
+                                                     glusterd_shd_start);
+        if (ret == -EINVAL)
+                ret = 0;
+        return ret;
+}
+
+int
+glusterd_nodesvcs_batch_op (glusterd_volinfo_t *volinfo,
+                             int (*nfs_op) (), int (*shd_op) ())
+{
+        int     ret = 0;
+
+        ret = nfs_op ();
+        if (ret)
+                goto out;
+
+        if (volinfo && !glusterd_is_volume_replicate (volinfo))
+                goto out;
+
+        ret = shd_op ();
+        if (ret)
+                goto out;
+out:
+        return ret;
+}
+
+int
+glusterd_nodesvcs_start (glusterd_volinfo_t *volinfo)
+{
+        return glusterd_nodesvcs_batch_op (volinfo,
+                                           glusterd_nfs_server_start,
+                                           glusterd_shd_start);
+}
+
+int
+glusterd_nodesvcs_stop (glusterd_volinfo_t *volinfo)
+{
+        return glusterd_nodesvcs_batch_op (volinfo,
+                                            glusterd_nfs_server_stop,
+                                            glusterd_shd_stop);
+}
+
+gf_boolean_t
+glusterd_are_all_volumes_stopped ()
+{
+        glusterd_conf_t                         *priv = NULL;
+        xlator_t                                *this = NULL;
+        glusterd_volinfo_t                      *voliter = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (voliter->status == GLUSTERD_STATUS_STARTED)
+                        return _gf_false;
+        }
+
+        return _gf_true;
+
+}
+
+gf_boolean_t
+glusterd_all_replicate_volumes_stopped ()
+{
+        glusterd_conf_t                         *priv = NULL;
+        xlator_t                                *this = NULL;
+        glusterd_volinfo_t                      *voliter = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (!glusterd_is_volume_replicate (voliter))
+                        continue;
+                if (voliter->status == GLUSTERD_STATUS_STARTED)
+                        return _gf_false;
+        }
+
+        return _gf_true;
+}
+
+int
+glusterd_nodesvcs_handle_graph_change (glusterd_volinfo_t *volinfo)
+{
+        int (*shd_op) () = NULL;
+        int (*nfs_op) () = NULL;
+
+        shd_op = glusterd_check_generate_start_shd;
+        nfs_op = glusterd_check_generate_start_nfs;
+        if (glusterd_are_all_volumes_stopped ()) {
+                shd_op = glusterd_shd_stop;
+                nfs_op = glusterd_nfs_server_stop;
+        } else if (glusterd_all_replicate_volumes_stopped()) {
+                shd_op = glusterd_shd_stop;
+        }
+        return glusterd_nodesvcs_batch_op (volinfo, nfs_op, shd_op);
+}
+
+int
+glusterd_nodesvcs_handle_reconfigure (glusterd_volinfo_t *volinfo)
+{
+        return glusterd_nodesvcs_batch_op (volinfo,
+                                           glusterd_reconfigure_nfs,
+                                           glusterd_reconfigure_shd);
+}
+
+int
+glusterd_volume_count_get (void)
+{
+        glusterd_volinfo_t      *tmp_volinfo = NULL;
+        int32_t                 ret = 0;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *priv = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+
+        list_for_each_entry (tmp_volinfo, &priv->volumes, vol_list) {
+                ret++;
+        }
+
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+int
+glusterd_brickinfo_get (uuid_t uuid, char *hostname, char *path,
+                        glusterd_brickinfo_t **brickinfo)
+{
+        glusterd_volinfo_t              *volinfo     = NULL;
+        glusterd_conf_t                 *priv = NULL;
+        xlator_t                        *this = NULL;
+        int                             ret = -1;
+
+        GF_ASSERT (path);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+
+                ret = glusterd_volume_brickinfo_get (uuid, hostname, path,
+                                                     volinfo, brickinfo);
+                if (ret == 0)
+                        /*Found*/
+                        goto out;
+        }
+out:
+        return ret;
+}
+
+int
+glusterd_brick_start (glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *brickinfo,
+                      gf_boolean_t wait)
+{
+        int                                     ret   = -1;
+        xlator_t                                *this = NULL;
+        glusterd_conf_t                         *conf = NULL;
+
+        if ((!brickinfo) || (!volinfo))
+                goto out;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (uuid_is_null (brickinfo->uuid)) {
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brickinfo->hostname, brickinfo->path);
+                        goto out;
+                }
+        }
+
+        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
+                ret = 0;
+                goto out;
+        }
+        ret = glusterd_volume_start_glusterfs (volinfo, brickinfo, wait);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to start brick %s:%s",
+                        brickinfo->hostname, brickinfo->path);
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
+        return ret;
+}
+
+int
+glusterd_restart_bricks (glusterd_conf_t *conf)
+{
+        int                   ret            = 0;
+        glusterd_volinfo_t   *volinfo        = NULL;
+        glusterd_brickinfo_t *brickinfo      = NULL;
+        glusterd_snap_t      *snap           = NULL;
+        gf_boolean_t          start_nodesvcs = _gf_false;
+        xlator_t             *this           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                        continue;
+                start_nodesvcs = _gf_true;
+                gf_log (this->name, GF_LOG_DEBUG, "starting the volume %s",
+                        volinfo->volname);
+                list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                        glusterd_brick_start (volinfo, brickinfo, _gf_false);
+                }
+        }
+
+        list_for_each_entry (snap, &conf->snapshots, snap_list) {
+                list_for_each_entry (volinfo, &snap->volumes, vol_list) {
+                        if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                                continue;
+                        start_nodesvcs = _gf_true;
+                        gf_log (this->name, GF_LOG_DEBUG, "starting the snap "
+                                "volume %s", volinfo->volname);
+                        list_for_each_entry (brickinfo, &volinfo->bricks,
+                                             brick_list) {
+                                glusterd_brick_start (volinfo, brickinfo,
+                                                      _gf_false);
+                        }
+                }
+        }
+
+        if (start_nodesvcs)
+                glusterd_nodesvcs_handle_graph_change (NULL);
+
+        return ret;
+}
+
+int
+_local_gsyncd_start (dict_t *this, char *key, data_t *value, void *data)
+{
+        char                        *path_list = NULL;
+        char                        *slave = NULL;
+        int                          uuid_len = 0;
+        int                          ret = 0;
+        char                         uuid_str[64] = {0};
+        glusterd_volinfo_t           *volinfo = NULL;
+        char                         *conf_path = NULL;
+
+        volinfo = data;
+        GF_ASSERT (volinfo);
+        slave = strchr(value->data, ':');
+        if (slave)
+                slave ++;
+        else
+                return 0;
+        uuid_len = (slave - value->data - 1);
+
+        strncpy (uuid_str, (char*)value->data, uuid_len);
+
+        ret = glusterd_get_local_brickpaths (volinfo, &path_list);
+
+        ret = dict_get_str (this, "conf_path", &conf_path);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Unable to fetch conf file path.");
+                goto out;
+        }
+
+        glusterd_start_gsync (volinfo, slave, path_list, conf_path,
+                              uuid_str, NULL);
+
+        GF_FREE (path_list);
+        path_list = NULL;
+
+out:
+        return ret;
+}
+
+int
+glusterd_volume_restart_gsyncds (glusterd_volinfo_t *volinfo)
+{
+        GF_ASSERT (volinfo);
+
+        dict_foreach (volinfo->gsync_slaves, _local_gsyncd_start, volinfo);
+        return 0;
+}
+
+int
+glusterd_restart_gsyncds (glusterd_conf_t *conf)
+{
+        glusterd_volinfo_t       *volinfo = NULL;
+        int                      ret = 0;
+
+        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                glusterd_volume_restart_gsyncds (volinfo);
+        }
+        return ret;
+}
+
+inline int
+glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo)
+{
+    int rcount = volinfo->replica_count;
+    int scount = volinfo->stripe_count;
+
+    return (rcount ? rcount : 1) * (scount ? scount : 1);
+}
+
+int
+glusterd_get_brickinfo (xlator_t *this, const char *brickname, int port,
+                        gf_boolean_t localhost, glusterd_brickinfo_t **brickinfo)
+{
+        glusterd_conf_t         *priv = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
+        int                     ret = -1;
+
+        GF_ASSERT (brickname);
+        GF_ASSERT (this);
+
+        priv = this->private;
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                list_for_each_entry (tmpbrkinfo, &volinfo->bricks,
+                                     brick_list) {
+                        if (localhost && !gf_is_local_addr (tmpbrkinfo->hostname))
+                                continue;
+                        if (!strcmp(tmpbrkinfo->path, brickname) &&
+                            (tmpbrkinfo->port == port)) {
+                                *brickinfo = tmpbrkinfo;
+                                return 0;
+                        }
+                }
+        }
+        return ret;
+}
+
+glusterd_brickinfo_t*
+glusterd_get_brickinfo_by_position (glusterd_volinfo_t *volinfo, uint32_t pos)
+{
+        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
+
+        list_for_each_entry (tmpbrkinfo, &volinfo->bricks,
+                             brick_list) {
+                if (pos == 0)
+                        return tmpbrkinfo;
+                pos--;
+        }
+        return NULL;
+}
+
+void
+glusterd_set_brick_status (glusterd_brickinfo_t  *brickinfo,
+                           gf_brick_status_t status)
+{
+        GF_ASSERT (brickinfo);
+        brickinfo->status = status;
+        if (GF_BRICK_STARTED == status) {
+                gf_log ("glusterd", GF_LOG_DEBUG, "Setting brick %s:%s status "
+                        "to started", brickinfo->hostname, brickinfo->path);
+        } else {
+                gf_log ("glusterd", GF_LOG_DEBUG, "Setting brick %s:%s status "
+                        "to stopped", brickinfo->hostname, brickinfo->path);
+        }
+}
+
+gf_boolean_t
+glusterd_is_brick_started (glusterd_brickinfo_t  *brickinfo)
+{
+        GF_ASSERT (brickinfo);
+        return (brickinfo->status == GF_BRICK_STARTED);
+}
+
+int
+glusterd_friend_brick_belongs (glusterd_volinfo_t *volinfo,
+                               glusterd_brickinfo_t *brickinfo, void* uuid)
+{
+        int             ret = -1;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (uuid);
+
+        if (uuid_is_null (brickinfo->uuid)) {
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret) {
+                        GF_ASSERT (0);
+                        goto out;
+                }
+        }
+        if (!uuid_compare (brickinfo->uuid, *((uuid_t *)uuid)))
+                return 0;
+out:
+        return -1;
+}
+
+#ifdef GF_LINUX_HOST_OS
+int
+glusterd_get_brick_root (char *path, char **mount_point)
+{
+        char           *ptr            = NULL;
+        char           *mnt_pt         = NULL;
+        struct stat     brickstat      = {0};
+        struct stat     buf            = {0};
+
+        if (!path)
+                goto err;
+        mnt_pt = gf_strdup (path);
+        if (!mnt_pt)
+                goto err;
+        if (stat (mnt_pt, &brickstat))
+                goto err;
+
+        while ((ptr = strrchr (mnt_pt, '/')) &&
+               ptr != mnt_pt) {
+
+                *ptr = '\0';
+                if (stat (mnt_pt, &buf)) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "error in "
+                                "stat: %s", strerror (errno));
+                        goto err;
+                }
+
+                if (brickstat.st_dev != buf.st_dev) {
+                        *ptr = '/';
+                        break;
+                }
+        }
+
+        if (ptr == mnt_pt) {
+                if (stat ("/", &buf)) {
+                        gf_log (THIS->name, GF_LOG_ERROR, "error in "
+                                "stat: %s", strerror (errno));
+                        goto err;
+                }
+                if (brickstat.st_dev == buf.st_dev)
+                        strcpy (mnt_pt, "/");
+        }
+
+        *mount_point = mnt_pt;
+        return 0;
+
+ err:
+        GF_FREE (mnt_pt);
+        return -1;
+}
+
+static char*
+glusterd_parse_inode_size (char *stream, char *pattern)
+{
+        char *needle = NULL;
+        char *trail  = NULL;
+
+        needle = strstr (stream, pattern);
+        if (!needle)
+                goto out;
+
+        needle = nwstrtail (needle, pattern);
+
+        trail = needle;
+        while (trail && isdigit (*trail)) trail++;
+        if (trail)
+                *trail = '\0';
+
+out:
+        return needle;
+}
+
+static int
+glusterd_add_inode_size_to_dict (dict_t *dict, int count)
+{
+        int             ret               = -1;
+        char            key[1024]         = {0};
+        char            buffer[4096]      = {0};
+        char           *inode_size        = NULL;
+        char           *device            = NULL;
+        char           *fs_name           = NULL;
+        char           *cur_word          = NULL;
+        char           *pattern           = NULL;
+        char           *trail             = NULL;
+        runner_t        runner            = {0, };
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.device", count);
+        ret = dict_get_str (dict, key, &device);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.fs_name", count);
+        ret = dict_get_str (dict, key, &fs_name);
+        if (ret)
+                goto out;
+
+        runinit (&runner);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        /* get inode size for xfs or ext2/3/4 */
+        if (!strcmp (fs_name, "xfs")) {
+
+                runner_add_args (&runner, "xfs_info", device, NULL);
+                pattern = "isize=";
+
+        } else if (IS_EXT_FS(fs_name)) {
+
+                runner_add_args (&runner, "tune2fs", "-l", device, NULL);
+                pattern = "Inode size:";
+
+        } else {
+                ret = 0;
+                gf_log (THIS->name, GF_LOG_INFO, "Skipped fetching "
+                        "inode size for %s: FS type not recommended",
+                        fs_name);
+                goto out;
+        }
+
+        ret = runner_start (&runner);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "could not get inode "
+                        "size for %s : %s package missing", fs_name,
+                        ((strcmp (fs_name, "xfs")) ?
+                         "e2fsprogs" : "xfsprogs"));
+                goto out;
+        }
+
+        for (;;) {
+                if (fgets (buffer, sizeof (buffer),
+                    runner_chio (&runner, STDOUT_FILENO)) == NULL)
+                        break;
+                trail = strrchr (buffer, '\n');
+                if (trail)
+                        *trail = '\0';
+
+                cur_word = glusterd_parse_inode_size (buffer, pattern);
+                if (cur_word)
+                        break;
+        }
+
+        ret = runner_end (&runner);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "%s exited with non-zero "
+                        "exit status", ((!strcmp (fs_name, "xfs")) ?
+                        "xfs_info" : "tune2fs"));
+                goto out;
+        }
+        if (!cur_word) {
+                ret = -1;
+                gf_log (THIS->name, GF_LOG_ERROR, "Unable to retrieve inode "
+                        "size using %s",
+                        (!strcmp (fs_name, "xfs")? "xfs_info": "tune2fs"));
+                goto out;
+        }
+
+        inode_size = gf_strdup (cur_word);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "brick%d.inode_size", count);
+
+        ret = dict_set_dynstr (dict, key, inode_size);
+
+ out:
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "failed to get inode size");
+        return ret;
+}
+
+struct mntent *
+glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab)
+{
+        struct mntent  *entry                = NULL;
+
+        mtab = setmntent (_PATH_MOUNTED, "r");
+        if (!mtab)
+                goto out;
+
+        entry = getmntent (mtab);
+
+        while (1) {
+                if (!entry)
+                        goto out;
+
+                if (!strcmp (entry->mnt_dir, mnt_pt) &&
+                    strcmp (entry->mnt_type, "rootfs"))
+                        break;
+                entry = getmntent (mtab);
+        }
+
+out:
+        return entry;
+}
+
+static int
+glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
+                                  dict_t *dict, int count)
+{
+        int             ret                  = -1;
+        char            key[1024]            = {0};
+        char            base_key[1024]       = {0};
+        char           *mnt_pt               = NULL;
+        char           *fs_name              = NULL;
+        char           *mnt_options          = NULL;
+        char           *device               = NULL;
+        struct mntent  *entry                = NULL;
+        FILE           *mtab                 = NULL;
+
+        snprintf (base_key, sizeof (base_key), "brick%d", count);
+
+        ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
+        if (ret)
+                goto out;
+
+        entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+        if (!entry) {
+                ret = -1;
+                goto out;
+        }
+
+        /* get device file */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.device", base_key);
+
+        device = gf_strdup (entry->mnt_fsname);
+        ret = dict_set_dynstr (dict, key, device);
+        if (ret)
+                goto out;
+
+        /* fs type */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.fs_name", base_key);
+
+        fs_name = gf_strdup (entry->mnt_type);
+        ret = dict_set_dynstr (dict, key, fs_name);
+        if (ret)
+                goto out;
+
+        /* mount options */
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.mnt_options", base_key);
+
+        mnt_options = gf_strdup (entry->mnt_opts);
+        ret = dict_set_dynstr (dict, key, mnt_options);
+
+ out:
+        GF_FREE (mnt_pt);
+        if (mtab)
+                endmntent (mtab);
+
+        return ret;
+}
+
+char*
+glusterd_get_brick_mount_details (glusterd_brickinfo_t *brickinfo)
+{
+        int             ret                  = -1;
+        char           *mnt_pt               = NULL;
+        char           *device               = NULL;
+        FILE           *mtab                 = NULL;
+        struct mntent  *entry                = NULL;
+        xlator_t       *this                 = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (brickinfo);
+
+        ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get mount point "
+                        "for %s brick", brickinfo->path);
+                goto out;
+        }
+
+        entry = glusterd_get_mnt_entry_info (mnt_pt, mtab);
+        if (NULL == entry) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get mnt entry "
+                        "for %s mount path", mnt_pt);
+                goto out;
+        }
+
+        /* get the fs_name/device */
+        device = gf_strdup (entry->mnt_fsname);
+
+out:
+        if (NULL != mtab) {
+                endmntent (mtab);
+        }
+
+        return device;
+}
+#endif
+
+int
+glusterd_add_brick_detail_to_dict (glusterd_volinfo_t *volinfo,
+                                   glusterd_brickinfo_t *brickinfo,
+                                   dict_t *dict, int count)
+{
+        int             ret               = -1;
+        uint64_t        memtotal          = 0;
+        uint64_t        memfree           = 0;
+        uint64_t        inodes_total      = 0;
+        uint64_t        inodes_free       = 0;
+        uint64_t        block_size        = 0;
+        char            key[1024]         = {0};
+        char            base_key[1024]    = {0};
+        struct statvfs  brickstat         = {0};
+        xlator_t       *this              = NULL;
+
+        this = THIS;
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (dict);
+
+        snprintf (base_key, sizeof (base_key), "brick%d", count);
+
+        ret = statvfs (brickinfo->path, &brickstat);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "statfs error: %s ",
+                        strerror (errno));
+                goto out;
+        }
+
+        /* file system block size */
+        block_size = brickstat.f_bsize;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.block_size", base_key);
+        ret = dict_set_uint64 (dict, key, block_size);
+        if (ret)
+                goto out;
+
+        /* free space in brick */
+        memfree = brickstat.f_bfree * brickstat.f_bsize;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.free", base_key);
+        ret = dict_set_uint64 (dict, key, memfree);
+        if (ret)
+                goto out;
+
+        /* total space of brick */
+        memtotal = brickstat.f_blocks * brickstat.f_bsize;
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.total", base_key);
+        ret = dict_set_uint64 (dict, key, memtotal);
+        if (ret)
+                goto out;
+
+        /* inodes: total and free counts only for ext2/3/4 and xfs */
+        inodes_total = brickstat.f_files;
+        if (inodes_total) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.total_inodes", base_key);
+                ret = dict_set_uint64 (dict, key, inodes_total);
+                if (ret)
+                        goto out;
+        }
+
+        inodes_free = brickstat.f_ffree;
+        if (inodes_free) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "%s.free_inodes", base_key);
+                ret = dict_set_uint64 (dict, key, inodes_free);
+                if (ret)
+                        goto out;
+        }
+#ifdef GF_LINUX_HOST_OS
+        ret = glusterd_add_brick_mount_details (brickinfo, dict, count);
+        if (ret)
+                goto out;
+
+        ret = glusterd_add_inode_size_to_dict (dict, count);
+#endif
+ out:
+        if (ret)
+                gf_log (this->name, GF_LOG_DEBUG, "Error adding brick"
+                        " detail to dict: %s", strerror (errno));
+        return ret;
+}
+
+int32_t
+glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
+                            glusterd_brickinfo_t *brickinfo,
+                            dict_t  *dict, int32_t count)
+{
+
+        int             ret                   = -1;
+        int32_t         pid                   = -1;
+        int32_t         brick_online          = -1;
+        char            key[1024]             = {0};
+        char            base_key[1024]        = {0};
+        char            pidfile[PATH_MAX]     = {0};
+        xlator_t        *this                 = NULL;
+        glusterd_conf_t *priv                 = NULL;
+
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+        GF_ASSERT (dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+
+        snprintf (base_key, sizeof (base_key), "brick%d", count);
+        snprintf (key, sizeof (key), "%s.hostname", base_key);
+
+        ret = dict_set_str (dict, key, brickinfo->hostname);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.path", base_key);
+        ret = dict_set_str (dict, key, brickinfo->path);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.port", base_key);
+        ret = dict_set_int32 (dict, key, brickinfo->port);
+        if (ret)
+                goto out;
+
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
+
+        brick_online = glusterd_is_service_running (pidfile, &pid);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.pid", base_key);
+        ret = dict_set_int32 (dict, key, pid);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%s.status", base_key);
+        ret = dict_set_int32 (dict, key, brick_online);
+
+out:
+        if (ret)
+                gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int32_t
+glusterd_get_all_volnames (dict_t *dict)
+{
+        int                    ret        = -1;
+        int32_t                vol_count  = 0;
+        char                   key[256]   = {0};
+        glusterd_volinfo_t    *entry      = NULL;
+        glusterd_conf_t       *priv       = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (entry, &priv->volumes, vol_list) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "vol%d", vol_count);
+                ret = dict_set_str (dict, key, entry->volname);
+                if (ret)
+                        goto out;
+
+                vol_count++;
+        }
+
+        ret = dict_set_int32 (dict, "vol_count", vol_count);
+
+ out:
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "failed to get all "
+                        "volume names for status");
+        return ret;
+}
+
+int
+glusterd_all_volume_cond_check (glusterd_condition_func func, int status,
+                                void *ctx)
+{
+        glusterd_conf_t         *priv = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        int                     ret = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        priv = this->private;
+
+        list_for_each_entry (volinfo, &priv->volumes, vol_list) {
+                list_for_each_entry (brickinfo, &volinfo->bricks,
+                                     brick_list) {
+                        ret = func (volinfo, brickinfo, ctx);
+                        if (ret != status) {
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_friend_find_by_uuid (uuid_t uuid,
+                              glusterd_peerinfo_t  **peerinfo)
+{
+        int                     ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_peerinfo_t     *entry = NULL;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (peerinfo);
+
+        *peerinfo = NULL;
+        priv    = this->private;
+
+        GF_ASSERT (priv);
+
+        if (uuid_is_null (uuid))
+                return -1;
+
+        list_for_each_entry (entry, &priv->peers, uuid_list) {
+                if (!uuid_compare (entry->uuid, uuid)) {
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                 "Friend found... state: %s",
+                        glusterd_friend_sm_state_name_get (entry->state.state));
+                        *peerinfo = entry;
+                        return 0;
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Friend with uuid: %s, not found",
+                uuid_utoa (uuid));
+        return ret;
+}
+
+
+int
+glusterd_friend_find_by_hostname (const char *hoststr,
+                                  glusterd_peerinfo_t  **peerinfo)
+{
+        int                     ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_peerinfo_t     *entry = NULL;
+        struct addrinfo         *addr = NULL;
+        struct addrinfo         *p = NULL;
+        char                    *host = NULL;
+        struct sockaddr_in6     *s6 = NULL;
+        struct sockaddr_in      *s4 = NULL;
+        struct in_addr          *in_addr = NULL;
+        char                    hname[1024] = {0,};
+        xlator_t                *this  = NULL;
+
+
+        this = THIS;
+        GF_ASSERT (hoststr);
+        GF_ASSERT (peerinfo);
+
+        *peerinfo = NULL;
+        priv    = this->private;
+
+        GF_ASSERT (priv);
+
+        list_for_each_entry (entry, &priv->peers, uuid_list) {
+                if (!strncasecmp (entry->hostname, hoststr,
+                                  1024)) {
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                 "Friend %s found.. state: %d", hoststr,
+                                  entry->state.state);
+                        *peerinfo = entry;
+                        return 0;
+                }
+        }
+
+        ret = getaddrinfo (hoststr, NULL, NULL, &addr);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error in getaddrinfo: %s\n",
+                        gai_strerror(ret));
+                goto out;
+        }
+
+        for (p = addr; p != NULL; p = p->ai_next) {
+                switch (p->ai_family) {
+                        case AF_INET:
+                                s4 = (struct sockaddr_in *) p->ai_addr;
+                                in_addr = &s4->sin_addr;
+                                break;
+                        case AF_INET6:
+                                s6 = (struct sockaddr_in6 *) p->ai_addr;
+                                in_addr =(struct in_addr *) &s6->sin6_addr;
+                                break;
+                       default: ret = -1;
+                                goto out;
+                }
+                host = inet_ntoa(*in_addr);
+
+                ret = getnameinfo (p->ai_addr, p->ai_addrlen, hname,
+                                   1024, NULL, 0, 0);
+                if (ret)
+                        goto out;
+
+                list_for_each_entry (entry, &priv->peers, uuid_list) {
+                        if (!strncasecmp (entry->hostname, host,
+                            1024) || !strncasecmp (entry->hostname,hname,
+                            1024)) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "Friend %s found.. state: %d",
+                                        hoststr, entry->state.state);
+                                *peerinfo = entry;
+                                freeaddrinfo (addr);
+                                return 0;
+                        }
+                }
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Unable to find friend: %s", hoststr);
+        if (addr)
+                freeaddrinfo (addr);
+        return -1;
+}
+
+int
+glusterd_hostname_to_uuid (char *hostname, uuid_t uuid)
+{
+        GF_ASSERT (hostname);
+        GF_ASSERT (uuid);
+
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        glusterd_conf_t         *priv = NULL;
+        int                     ret = -1;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = glusterd_friend_find_by_hostname (hostname, &peerinfo);
+        if (ret) {
+                if (gf_is_local_addr (hostname)) {
+                        uuid_copy (uuid, MY_UUID);
+                        ret = 0;
+                } else {
+                        ret = 0;
+                        goto out;
+                }
+        } else {
+                uuid_copy (uuid, peerinfo->uuid);
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_brick_stop (glusterd_volinfo_t *volinfo,
+                     glusterd_brickinfo_t *brickinfo,
+                     gf_boolean_t del_brick)
+{
+        int                                     ret   = -1;
+        xlator_t                                *this = NULL;
+        glusterd_conf_t                         *conf = NULL;
+
+        if ((!brickinfo) || (!volinfo))
+                goto out;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (uuid_is_null (brickinfo->uuid)) {
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brickinfo->hostname, brickinfo->path);
+                        goto out;
+                }
+        }
+
+        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
+                ret = 0;
+                if (del_brick)
+                        glusterd_delete_brick (volinfo, brickinfo);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "About to stop glusterfs"
+                " for brick %s:%s", brickinfo->hostname,
+                brickinfo->path);
+        ret = glusterd_volume_stop_glusterfs (volinfo, brickinfo, del_brick);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Unable to stop"
+                        " brick: %s:%s", brickinfo->hostname,
+                        brickinfo->path);
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
+        return ret;
+}
+
+int
+glusterd_is_defrag_on (glusterd_volinfo_t *volinfo)
+{
+        return (volinfo->rebal.defrag != NULL);
+}
+
+gf_boolean_t
+glusterd_is_rb_ongoing (glusterd_volinfo_t *volinfo)
+{
+        gf_boolean_t     ret = _gf_false;
+
+        GF_ASSERT (volinfo);
+
+        if (glusterd_is_rb_started (volinfo) ||
+            glusterd_is_rb_paused (volinfo))
+                ret = _gf_true;
+
+        return ret;
+}
+
+int
+glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
+                             char *op_errstr, size_t len)
+{
+        glusterd_brickinfo_t    *newbrickinfo = NULL;
+        int                     ret = -1;
+        gf_boolean_t            is_allocated = _gf_false;
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        glusterd_conf_t         *priv = NULL;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+
+        GF_ASSERT (brick);
+        GF_ASSERT (op_errstr);
+
+        if (!brickinfo) {
+                ret = glusterd_brickinfo_new_from_brick (brick, &newbrickinfo);
+                if (ret)
+                        goto out;
+                is_allocated = _gf_true;
+        } else {
+                newbrickinfo = brickinfo;
+        }
+
+        ret = glusterd_resolve_brick (newbrickinfo);
+        if (ret) {
+                snprintf(op_errstr, len, "Host %s is not in \'Peer "
+                         "in Cluster\' state", newbrickinfo->hostname);
+                goto out;
+        }
+
+        if (!uuid_compare (MY_UUID, newbrickinfo->uuid)) {
+                /* brick is local */
+                if (!glusterd_is_brickpath_available (newbrickinfo->uuid,
+                                                      newbrickinfo->path)) {
+                        snprintf(op_errstr, len, "Brick: %s not available."
+                                 " Brick may be containing or be contained "
+                                 "by an existing brick", brick);
+                        ret = -1;
+                        goto out;
+                }
+
+        } else {
+                ret = glusterd_friend_find_by_uuid (newbrickinfo->uuid,
+                                                    &peerinfo);
+                if (ret) {
+                        snprintf (op_errstr, len, "Failed to find host %s",
+                                  newbrickinfo->hostname);
+                        goto out;
+                }
+
+                if ((!peerinfo->connected)) {
+                        snprintf(op_errstr, len, "Host %s not connected",
+                                 newbrickinfo->hostname);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) {
+                        snprintf(op_errstr, len, "Host %s is not in \'Peer "
+                                 "in Cluster\' state",
+                                 newbrickinfo->hostname);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+out:
+        if (is_allocated)
+                glusterd_brickinfo_delete (newbrickinfo);
+        if (op_errstr[0] != '\0')
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
+        return ret;
+}
+
+int
+glusterd_is_rb_started(glusterd_volinfo_t *volinfo)
+{
+        gf_log ("", GF_LOG_DEBUG,
+                "is_rb_started:status=%d", volinfo->rep_brick.rb_status);
+        return (volinfo->rep_brick.rb_status == GF_RB_STATUS_STARTED);
+
+}
+
+int
+glusterd_is_rb_paused ( glusterd_volinfo_t *volinfo)
+{
+        gf_log ("", GF_LOG_DEBUG,
+                "is_rb_paused:status=%d", volinfo->rep_brick.rb_status);
+
+        return (volinfo->rep_brick.rb_status == GF_RB_STATUS_PAUSED);
+}
+
+inline int
+glusterd_set_rb_status (glusterd_volinfo_t *volinfo, gf_rb_status_t status)
+{
+        gf_log ("", GF_LOG_DEBUG,
+                "setting status from %d to %d",
+                volinfo->rep_brick.rb_status,
+                status);
+
+        volinfo->rep_brick.rb_status = status;
+        return 0;
+}
+
+inline int
+glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
+                          glusterd_brickinfo_t *src, glusterd_brickinfo_t *dst)
+{
+        glusterd_replace_brick_t        *rb = NULL;
+
+        GF_ASSERT (volinfo);
+
+        rb = &volinfo->rep_brick;
+
+        if (!rb->src_brick || !rb->dst_brick)
+                return -1;
+
+        if (strcmp (rb->src_brick->hostname, src->hostname) ||
+            strcmp (rb->src_brick->path, src->path)) {
+                gf_log("", GF_LOG_ERROR, "Replace brick src bricks differ");
+                return -1;
+        }
+
+        if (strcmp (rb->dst_brick->hostname, dst->hostname) ||
+            strcmp (rb->dst_brick->path, dst->path)) {
+                gf_log ("", GF_LOG_ERROR, "Replace brick dst bricks differ");
+                return -1;
+        }
+
+        return 0;
+}
+
+/*path needs to be absolute; works only on gfid, volume-id*/
+static int
+glusterd_is_uuid_present (char *path, char *xattr, gf_boolean_t *present)
+{
+        GF_ASSERT (path);
+        GF_ASSERT (xattr);
+        GF_ASSERT (present);
+
+        int     ret      = -1;
+        uuid_t  uid     = {0,};
+
+        if (!path || !xattr || !present)
+                goto out;
+
+        ret = sys_lgetxattr (path, xattr, &uid, 16);
+
+        if (ret >= 0) {
+                *present = _gf_true;
+                ret = 0;
+                goto out;
+        }
+
+        switch (errno) {
+#if defined(ENODATA)
+                case ENODATA: /* FALLTHROUGH */
+#endif
+#if defined(ENOATTR) && (ENOATTR != ENODATA)
+                case ENOATTR: /* FALLTHROUGH */
+#endif
+                case ENOTSUP:
+                        *present = _gf_false;
+                        ret = 0;
+                        break;
+                default:
+                        break;
+        }
+out:
+        return ret;
+}
+
+/*path needs to be absolute*/
+static int
+glusterd_is_path_in_use (char *path, gf_boolean_t *in_use, char **op_errstr)
+{
+        int             i               = 0;
+        int             ret             = -1;
+        gf_boolean_t    used            = _gf_false;
+        char            dir[PATH_MAX]   = {0,};
+        char            *curdir         = NULL;
+        char            msg[2048]       = {0};
+        char            *keys[3]         = {GFID_XATTR_KEY,
+                                            GF_XATTR_VOL_ID_KEY,
+                                            NULL};
+
+        GF_ASSERT (path);
+        if (!path)
+                goto out;
+
+        strcpy (dir, path);
+        curdir = dir;
+        do {
+                for (i = 0; !used && keys[i]; i++) {
+                        ret = glusterd_is_uuid_present (curdir, keys[i], &used);
+                        if (ret)
+                                goto out;
+                }
+
+                if (used)
+                        break;
+
+                curdir = dirname (curdir);
+                if (!strcmp (curdir, "."))
+                        goto out;
+
+
+        } while (strcmp (curdir, "/"));
+
+        if (!strcmp (curdir, "/")) {
+                for (i = 0; !used && keys[i]; i++) {
+                        ret = glusterd_is_uuid_present (curdir, keys[i], &used);
+                        if (ret)
+                                goto out;
+                }
+        }
+
+        ret = 0;
+        *in_use = used;
+out:
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to get extended "
+                          "attribute %s, reason: %s", keys[i],
+                          strerror (errno));
+        }
+
+        if (*in_use) {
+                if (!strcmp (path, curdir)) {
+                        snprintf (msg, sizeof (msg), "%s is already part of a "
+                          "volume", path);
+                } else {
+                        snprintf (msg, sizeof (msg), "parent directory %s is "
+                          "already part of a volume", curdir);
+                }
+        }
+
+        if (strlen (msg)) {
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+
+        return ret;
+}
+
+int
+glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
+                                    char **op_errstr, gf_boolean_t is_force)
+{
+        int             ret             = -1;
+        char            msg[2048]       = {0,};
+        gf_boolean_t    in_use          = _gf_false;
+        int             flags           = 0;
+
+        /* Check for xattr support in backend fs */
+        ret = sys_lsetxattr (path, "trusted.glusterfs.test",
+                             "working", 8, 0);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Glusterfs is not"
+                          " supported on brick: %s:%s.\nSetting"
+                          " extended attributes failed, reason:"
+                          " %s.", host, path, strerror(errno));
+                goto out;
+
+        } else {
+                sys_lremovexattr (path, "trusted.glusterfs.test");
+        }
+
+        ret = glusterd_is_path_in_use (path, &in_use, op_errstr);
+        if (ret)
+                goto out;
+
+        if (in_use && !is_force) {
+                ret = -1;
+                goto out;
+        }
+
+
+        if (!is_force)
+                flags = XATTR_CREATE;
+
+        ret = sys_lsetxattr (path, GF_XATTR_VOL_ID_KEY, uuid, 16,
+                             flags);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to set extended "
+                          "attributes %s, reason: %s",
+                          GF_XATTR_VOL_ID_KEY, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (strlen (msg))
+                *op_errstr = gf_strdup (msg);
+
+        return ret;
+}
+
+int
+glusterd_sm_tr_log_transition_add_to_dict (dict_t *dict,
+                                           glusterd_sm_tr_log_t *log, int i,
+                                           int count)
+{
+        int     ret = -1;
+        char    key[512] = {0};
+	char    timestr[64] = {0,};
+        char    *str = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (log);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "log%d-old-state", count);
+        str = log->state_name_get (log->transitions[i].old_state);
+        ret = dict_set_str (dict, key, str);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "log%d-event", count);
+        str = log->event_name_get (log->transitions[i].event);
+        ret = dict_set_str (dict, key, str);
+        if (ret)
+                goto out;
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "log%d-new-state", count);
+        str = log->state_name_get (log->transitions[i].new_state);
+        ret = dict_set_str (dict, key, str);
+        if (ret)
+                goto out;
+
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "log%d-time", count);
+        gf_time_fmt (timestr, sizeof timestr, log->transitions[i].time,
+                     gf_timefmt_FT);
+        str = gf_strdup (timestr);
+        ret = dict_set_dynstr (dict, key, str);
+        if (ret)
+                goto out;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_sm_tr_log_add_to_dict (dict_t *dict,
+                                glusterd_sm_tr_log_t *circular_log)
+{
+        int     ret = -1;
+        int     i = 0;
+        int     start = 0;
+        int     end     = 0;
+        int     index = 0;
+        char    key[256] = {0};
+        glusterd_sm_tr_log_t *log = NULL;
+        int     count = 0;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (circular_log);
+
+        log = circular_log;
+        if (!log->count)
+                return 0;
+
+        if (log->count == log->size)
+                start = log->current + 1;
+
+        end = start + log->count;
+        for (i = start; i < end; i++, count++) {
+                index = i % log->count;
+                ret = glusterd_sm_tr_log_transition_add_to_dict (dict, log, index,
+                                                                 count);
+                if (ret)
+                        goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "count");
+        ret = dict_set_int32 (dict, key, log->count);
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_sm_tr_log_init (glusterd_sm_tr_log_t *log,
+                         char * (*state_name_get) (int),
+                         char * (*event_name_get) (int),
+                         size_t  size)
+{
+        glusterd_sm_transition_t *transitions = NULL;
+        int                      ret = -1;
+
+        GF_ASSERT (size > 0);
+        GF_ASSERT (log && state_name_get && event_name_get);
+
+        if (!log || !state_name_get || !event_name_get || (size <= 0))
+                goto out;
+
+        transitions = GF_CALLOC (size, sizeof (*transitions),
+                                 gf_gld_mt_sm_tr_log_t);
+        if (!transitions)
+                goto out;
+
+        log->transitions = transitions;
+        log->size        = size;
+        log->state_name_get = state_name_get;
+        log->event_name_get = event_name_get;
+        ret = 0;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+void
+glusterd_sm_tr_log_delete (glusterd_sm_tr_log_t *log)
+{
+        if (!log)
+                return;
+        GF_FREE (log->transitions);
+        return;
+}
+
+int
+glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
+                                   int old_state, int new_state,
+                                   int event)
+{
+        glusterd_sm_transition_t *transitions = NULL;
+        int                      ret = -1;
+        int                      next = 0;
+        xlator_t                 *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (log);
+        if (!log)
+                goto out;
+
+        transitions = log->transitions;
+        if (!transitions)
+                goto out;
+
+        if (log->count)
+                next = (log->current + 1) % log->size;
+        else
+                next = 0;
+
+        transitions[next].old_state = old_state;
+        transitions[next].new_state = new_state;
+        transitions[next].event     = event;
+        time (&transitions[next].time);
+        log->current = next;
+        if (log->count < log->size)
+                log->count++;
+        ret = 0;
+        gf_log (this->name, GF_LOG_DEBUG, "Transitioning from '%s' to '%s' "
+                "due to event '%s'", log->state_name_get (old_state),
+                log->state_name_get (new_state), log->event_name_get (event));
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
+                       glusterd_friend_sm_state_t state, uuid_t *uuid,
+                       const char *hostname, int port)
+{
+        glusterd_peerinfo_t      *new_peer = NULL;
+        int                      ret = -1;
+
+        GF_ASSERT (peerinfo);
+        if (!peerinfo)
+                goto out;
+
+        new_peer = GF_CALLOC (1, sizeof (*new_peer), gf_gld_mt_peerinfo_t);
+        if (!new_peer)
+                goto out;
+
+        new_peer->state.state = state;
+        if (hostname)
+                new_peer->hostname = gf_strdup (hostname);
+
+        INIT_LIST_HEAD (&new_peer->uuid_list);
+
+        if (uuid) {
+                uuid_copy (new_peer->uuid, *uuid);
+        }
+
+        ret = glusterd_sm_tr_log_init (&new_peer->sm_log,
+                                       glusterd_friend_sm_state_name_get,
+                                       glusterd_friend_sm_event_name_get,
+                                       GLUSTERD_TR_LOG_SIZE);
+        if (ret)
+                goto out;
+
+        if (new_peer->state.state == GD_FRIEND_STATE_BEFRIENDED)
+                new_peer->quorum_contrib = QUORUM_WAITING;
+        new_peer->port = port;
+        *peerinfo = new_peer;
+out:
+        if (ret && new_peer)
+                glusterd_friend_cleanup (new_peer);
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_peer_destroy (glusterd_peerinfo_t *peerinfo)
+{
+        int32_t                         ret = -1;
+
+        if (!peerinfo)
+                goto out;
+
+        ret = glusterd_store_delete_peerinfo (peerinfo);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Deleting peer info failed");
+        }
+
+        list_del_init (&peerinfo->uuid_list);
+        GF_FREE (peerinfo->hostname);
+        glusterd_sm_tr_log_delete (&peerinfo->sm_log);
+        GF_FREE (peerinfo);
+        peerinfo = NULL;
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+glusterd_remove_pending_entry (struct list_head *list, void *elem)
+{
+        glusterd_pending_node_t *pending_node = NULL;
+        glusterd_pending_node_t *tmp = NULL;
+        int                     ret = 0;
+
+        list_for_each_entry_safe (pending_node, tmp, list, list) {
+                if (elem == pending_node->node) {
+                        list_del_init (&pending_node->list);
+                        GF_FREE (pending_node);
+                        ret = 0;
+                        goto out;
+                }
+        }
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+
+}
+
+int
+glusterd_clear_pending_nodes (struct list_head *list)
+{
+        glusterd_pending_node_t *pending_node = NULL;
+        glusterd_pending_node_t *tmp = NULL;
+
+        list_for_each_entry_safe (pending_node, tmp, list, list) {
+                list_del_init (&pending_node->list);
+                GF_FREE (pending_node);
+        }
+
+        return 0;
+}
+
+gf_boolean_t
+glusterd_peerinfo_is_uuid_unknown (glusterd_peerinfo_t *peerinfo)
+{
+        GF_ASSERT (peerinfo);
+
+        if (uuid_is_null (peerinfo->uuid))
+                return _gf_true;
+        return _gf_false;
+}
+
+int32_t
+glusterd_delete_volume (glusterd_volinfo_t *volinfo)
+{
+        int             ret = -1;
+        GF_ASSERT (volinfo);
+
+        ret = glusterd_store_delete_volume (volinfo);
+
+        if (ret)
+                goto out;
+
+        ret = glusterd_volinfo_delete (volinfo);
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_delete_brick (glusterd_volinfo_t* volinfo,
+                       glusterd_brickinfo_t *brickinfo)
+{
+        int             ret = 0;
+        char      voldir[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = THIS->private;
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+
+        GLUSTERD_GET_VOLUME_DIR(voldir, volinfo, priv);
+
+        glusterd_delete_volfile (volinfo, brickinfo);
+        glusterd_store_delete_brick (brickinfo, voldir);
+        glusterd_brickinfo_delete (brickinfo);
+        volinfo->brick_count--;
+        return ret;
+}
+
+int32_t
+glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo)
+{
+        int             ret = 0;
+        glusterd_brickinfo_t *brickinfo = NULL;
+        glusterd_brickinfo_t *tmp = NULL;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks, brick_list) {
+                ret = glusterd_delete_brick (volinfo, brickinfo);
+        }
+        return ret;
+}
+
+int
+glusterd_get_local_brickpaths (glusterd_volinfo_t *volinfo, char **pathlist)
+{
+        char                 **path_tokens  = NULL;
+        char                  *tmp_path_list = NULL;
+        char                   path[PATH_MAX] = "";
+        int32_t                count          = 0;
+        int32_t                pathlen        = 0;
+        int32_t                total_len      = 0;
+        int32_t                ret            = 0;
+        int                    i              = 0;
+        glusterd_brickinfo_t  *brickinfo      = NULL;
+
+        if ((!volinfo) || (!pathlist))
+            goto out;
+
+        path_tokens = GF_CALLOC (sizeof(char*), volinfo->brick_count,
+                                 gf_gld_mt_charptr);
+        if (!path_tokens) {
+                gf_log ("", GF_LOG_DEBUG, "Could not allocate memory.");
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                 pathlen = snprintf (path, sizeof(path),
+                                     "--path=%s ", brickinfo->path);
+                 if (pathlen < sizeof(path))
+                        path[pathlen] = '\0';
+                 else
+                        path[sizeof(path)-1] = '\0';
+                 path_tokens[count] = gf_strdup (path);
+                 if (!path_tokens[count]) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not allocate memory.");
+                        ret = -1;
+                        goto out;
+                 }
+                 count++;
+                 total_len += pathlen;
+        }
+
+        tmp_path_list = GF_CALLOC (sizeof(char), total_len + 1,
+                                   gf_gld_mt_char);
+        if (!tmp_path_list) {
+                gf_log ("", GF_LOG_DEBUG, "Could not allocate memory.");
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; i < count; i++)
+                strcat (tmp_path_list, path_tokens[i]);
+
+        if (count)
+                *pathlist = tmp_path_list;
+
+        ret = count;
+out:
+        for (i = 0; i < count; i++) {
+                GF_FREE (path_tokens[i]);
+                path_tokens[i] = NULL;
+        }
+
+        GF_FREE (path_tokens);
+        path_tokens = NULL;
+
+        if (ret == 0) {
+                gf_log ("", GF_LOG_DEBUG, "No Local Bricks Present.");
+                GF_FREE (tmp_path_list);
+                tmp_path_list = NULL;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
+                      char *path_list, char *conf_path,
+                      char *glusterd_uuid_str,
+                      char **op_errstr)
+{
+        int32_t         ret     = 0;
+        int32_t         status  = 0;
+        char            uuid_str [64] = {0};
+        runner_t        runner = {0,};
+        xlator_t        *this = NULL;
+        glusterd_conf_t *priv = NULL;
+        int             errcode = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        uuid_utoa_r (MY_UUID, uuid_str);
+
+        if (!path_list) {
+                ret = 0;
+                gf_log ("", GF_LOG_DEBUG, "No Bricks in this node."
+                        " Not starting gsyncd.");
+                goto out;
+        }
+
+        ret = gsync_status (master_vol->volname, slave, conf_path, &status);
+        if (status == 0)
+                goto out;
+
+        uuid_utoa_r (master_vol->volume_id, uuid_str);
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
+                          path_list, "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
+        runner_argprintf (&runner, ":%s", master_vol->volname);
+        runner_add_args  (&runner, slave, "--config-set", "session-owner",
+                          uuid_str, NULL);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret == -1) {
+                errcode = -1;
+                goto out;
+        }
+
+        runinit (&runner);
+        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
+                          path_list, "--monitor", "-c", NULL);
+        runner_argprintf (&runner, "%s", conf_path);
+        runner_argprintf (&runner, ":%s", master_vol->volname);
+        runner_argprintf (&runner, "--glusterd-uuid=%s",
+                          uuid_utoa (priv->uuid));
+        runner_add_arg   (&runner, slave);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret == -1) {
+                gf_asprintf (op_errstr, GEOREP" start failed for %s %s",
+                             master_vol->volname, slave);
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if ((ret != 0) && errcode == -1) {
+                if (op_errstr)
+                        *op_errstr = gf_strdup ("internal error, cannot start "
+                                                "the " GEOREP " session");
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_recreate_volfiles (glusterd_conf_t *conf)
+{
+
+        glusterd_volinfo_t      *volinfo = NULL;
+        int                      ret = 0;
+        int                      op_ret = 0;
+
+        GF_ASSERT (conf);
+        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                ret = generate_brick_volfiles (volinfo);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                "regenerate brick volfiles for %s",
+                                volinfo->volname);
+                        op_ret = ret;
+                }
+                ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                "regenerate trusted client volfiles for %s",
+                                volinfo->volname);
+                        op_ret = ret;
+                }
+                ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
+                                "regenerate client volfiles for %s",
+                                volinfo->volname);
+                        op_ret = ret;
+                }
+        }
+        return op_ret;
+}
+
+int32_t
+glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf)
+{
+        int              ret                            = 0;
+        char            *type                           = NULL;
+        gf_boolean_t     upgrade                        = _gf_false;
+        gf_boolean_t     downgrade                      = _gf_false;
+        gf_boolean_t     regenerate_volfiles            = _gf_false;
+        gf_boolean_t     terminate                      = _gf_false;
+
+        ret = dict_get_str (options, "upgrade", &type);
+        if (!ret) {
+                ret = gf_string2boolean (type, &upgrade);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "upgrade option "
+                                "%s is not a valid boolean type", type);
+                        ret = -1;
+                        goto out;
+                }
+                if (_gf_true == upgrade)
+                        regenerate_volfiles = _gf_true;
+        }
+
+        ret = dict_get_str (options, "downgrade", &type);
+        if (!ret) {
+                ret = gf_string2boolean (type, &downgrade);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "downgrade option "
+                                "%s is not a valid boolean type", type);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (upgrade && downgrade) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Both upgrade and downgrade"
+                        " options are set. Only one should be on");
+                ret = -1;
+                goto out;
+        }
+
+        if (!upgrade && !downgrade)
+                ret = 0;
+        else
+                terminate = _gf_true;
+        if (regenerate_volfiles) {
+                ret = glusterd_recreate_volfiles (conf);
+        }
+out:
+        if (terminate && (ret == 0))
+                kill (getpid(), SIGTERM);
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_volume_replicate (glusterd_volinfo_t *volinfo)
+{
+        gf_boolean_t    replicates = _gf_false;
+        if (volinfo && ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) ||
+            (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE)))
+                replicates = _gf_true;
+        return replicates;
+}
+
+int
+glusterd_set_dump_options (char *dumpoptions_path, char *options,
+                           int option_cnt)
+{
+        int     ret = 0;
+        char    *dup_options = NULL;
+        char    *option = NULL;
+        char    *tmpptr = NULL;
+        FILE    *fp = NULL;
+        int     nfs_cnt = 0;
+
+        if (0 == option_cnt ||
+            (option_cnt == 1 && (!strcmp (options, "nfs ")))) {
+                ret = 0;
+                goto out;
+        }
+
+        fp = fopen (dumpoptions_path, "w");
+        if (!fp) {
+                ret = -1;
+                goto out;
+        }
+        dup_options = gf_strdup (options);
+        gf_log ("", GF_LOG_INFO, "Received following statedump options: %s",
+                dup_options);
+        option = strtok_r (dup_options, " ", &tmpptr);
+        while (option) {
+                if (!strcmp (option, "nfs")) {
+                        if (nfs_cnt > 0) {
+                                unlink (dumpoptions_path);
+                                ret = 0;
+                                goto out;
+                        }
+                        nfs_cnt++;
+                        option = strtok_r (NULL, " ", &tmpptr);
+                        continue;
+                }
+                fprintf (fp, "%s=yes\n", option);
+                option = strtok_r (NULL, " ", &tmpptr);
+        }
+
+out:
+        if (fp)
+                fclose (fp);
+        GF_FREE (dup_options);
+        return ret;
+}
+
+int
+glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
+                          glusterd_brickinfo_t *brickinfo,
+                          char *options, int option_cnt, char **op_errstr)
+{
+        int                     ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *conf = NULL;
+        char                    pidfile_path[PATH_MAX] = {0,};
+        char                    dumpoptions_path[PATH_MAX] = {0,};
+        FILE                    *pidfile = NULL;
+        pid_t                   pid = -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        if (uuid_is_null (brickinfo->uuid)) {
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR,
+                                "Cannot resolve brick %s:%s",
+                                brickinfo->hostname, brickinfo->path);
+                        goto out;
+                }
+        }
+
+        if (uuid_compare (brickinfo->uuid, MY_UUID)) {
+                ret = 0;
+                goto out;
+        }
+
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile_path, volinfo, brickinfo, conf);
+
+        pidfile = fopen (pidfile_path, "r");
+        if (!pidfile) {
+                gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
+                        pidfile_path);
+                ret = -1;
+                goto out;
+        }
+
+        ret = fscanf (pidfile, "%d", &pid);
+        if (ret <= 0) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get pid of brick process");
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (dumpoptions_path, sizeof (dumpoptions_path),
+                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
+        ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "error while parsing the statedump "
+                        "options");
+                ret = -1;
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_INFO, "Performing statedump on brick with pid %d",
+                pid);
+
+        kill (pid, SIGUSR1);
+
+        sleep (1);
+        ret = 0;
+out:
+        unlink (dumpoptions_path);
+        if (pidfile)
+                fclose (pidfile);
+        return ret;
+}
+
+int
+glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr)
+{
+        int                     ret = -1;
+        xlator_t                *this = NULL;
+        glusterd_conf_t         *conf = NULL;
+        char                    pidfile_path[PATH_MAX] = {0,};
+        char                    path[PATH_MAX] = {0,};
+        FILE                    *pidfile = NULL;
+        pid_t                   pid = -1;
+        char                    dumpoptions_path[PATH_MAX] = {0,};
+        char                    *option = NULL;
+        char                    *tmpptr = NULL;
+        char                    *dup_options = NULL;
+        char                    msg[256] = {0,};
+
+        this = THIS;
+        GF_ASSERT (this);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        dup_options = gf_strdup (options);
+        option = strtok_r (dup_options, " ", &tmpptr);
+        if (strcmp (option, "nfs")) {
+                snprintf (msg, sizeof (msg), "for nfs statedump, options should"
+                          " be after the key nfs");
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+        GLUSTERD_GET_NFS_DIR (path, conf);
+        GLUSTERD_GET_NFS_PIDFILE (pidfile_path, path);
+
+        pidfile = fopen (pidfile_path, "r");
+        if (!pidfile) {
+                gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
+                        pidfile_path);
+                ret = -1;
+                goto out;
+        }
+
+        ret = fscanf (pidfile, "%d", &pid);
+        if (ret <= 0) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get pid of brick process");
+                ret = -1;
+                goto out;
+        }
+
+        snprintf (dumpoptions_path, sizeof (dumpoptions_path),
+                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
+        ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_ERROR, "error while parsing the statedump "
+                        "options");
+                ret = -1;
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_INFO, "Performing statedump on nfs server with "
+                "pid %d", pid);
+
+        kill (pid, SIGUSR1);
+
+        sleep (1);
+
+        ret = 0;
+out:
+        if (pidfile)
+                fclose (pidfile);
+        unlink (dumpoptions_path);
+        GF_FREE (dup_options);
+        return ret;
+}
+
+/* Checks if the given peer contains all the bricks belonging to the
+ * given volume. Returns true if it does else returns false
+ */
+gf_boolean_t
+glusterd_friend_contains_vol_bricks (glusterd_volinfo_t *volinfo,
+                                     uuid_t friend_uuid)
+{
+        gf_boolean_t            ret = _gf_true;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+
+        GF_ASSERT (volinfo);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (uuid_compare (friend_uuid, brickinfo->uuid)) {
+                        ret = _gf_false;
+                        break;
+                }
+        }
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Remove all volumes which completely belong to given friend
+ */
+int
+glusterd_friend_remove_cleanup_vols (uuid_t uuid)
+{
+        int                     ret = -1;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_volinfo_t      *tmp_volinfo = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry_safe (volinfo, tmp_volinfo,
+                                  &priv->volumes, vol_list) {
+                if (glusterd_friend_contains_vol_bricks (volinfo, uuid)) {
+                        gf_log (THIS->name, GF_LOG_INFO,
+                                "Deleting stale volume %s", volinfo->volname);
+                        ret = glusterd_delete_volume (volinfo);
+                        if (ret) {
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "Error deleting stale volume");
+                                goto out;
+                        }
+                }
+        }
+        ret = 0;
+out:
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Check if the all peers are connected and befriended, except the peer
+ * specified (the peer being detached)
+ */
+gf_boolean_t
+glusterd_chk_peers_connected_befriended (uuid_t skip_uuid)
+{
+        gf_boolean_t            ret = _gf_true;
+        glusterd_peerinfo_t     *peerinfo = NULL;
+        glusterd_conf_t         *priv = NULL;
+
+        priv= THIS->private;
+        GF_ASSERT (priv);
+
+        list_for_each_entry (peerinfo, &priv->peers, uuid_list) {
+
+                if (!uuid_is_null (skip_uuid) && !uuid_compare (skip_uuid,
+                                                           peerinfo->uuid))
+                        continue;
+
+                if ((GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state)
+                    || !(peerinfo->connected)) {
+                        ret = _gf_false;
+                        break;
+                }
+        }
+        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %s",
+                (ret?"TRUE":"FALSE"));
+        return ret;
+}
+
+void
+glusterd_get_client_filepath (char *filepath, glusterd_volinfo_t *volinfo,
+                              gf_transport_type type)
+{
+        char  path[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+
+        if ((volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) &&
+            (type == GF_TRANSPORT_RDMA))
+                snprintf (filepath, PATH_MAX, "%s/%s.rdma-fuse.vol",
+                          path, volinfo->volname);
+        else
+                snprintf (filepath, PATH_MAX, "%s/%s-fuse.vol",
+                          path, volinfo->volname);
+}
+
+void
+glusterd_get_trusted_client_filepath (char *filepath,
+                                      glusterd_volinfo_t *volinfo,
+                                      gf_transport_type type)
+{
+        char  path[PATH_MAX] = {0,};
+        glusterd_conf_t *priv = NULL;
+
+        priv = THIS->private;
+
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+
+        if ((volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) &&
+            (type == GF_TRANSPORT_RDMA))
+                snprintf (filepath, PATH_MAX,
+                          "%s/trusted-%s.rdma-fuse.vol",
+                          path, volinfo->volname);
+        else
+                snprintf (filepath, PATH_MAX,
+                          "%s/trusted-%s-fuse.vol",
+                          path, volinfo->volname);
+}
+
+int
+glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
+                              size_t len, int cmd, defrag_cbk_fn_t cbk)
+{
+        glusterd_conf_t         *priv                   = NULL;
+        char                     pidfile[PATH_MAX];
+        int                      ret                    = -1;
+        pid_t                    pid;
+
+        priv = THIS->private;
+        if (!priv)
+                return ret;
+
+        GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
+
+        if (!glusterd_is_service_running (pidfile, &pid)) {
+                glusterd_handle_defrag_start (volinfo, op_errstr, len, cmd,
+                                              cbk, volinfo->rebal.op);
+        } else {
+                glusterd_rebalance_rpc_create (volinfo, priv, cmd);
+        }
+
+        return ret;
+}
+
+int
+glusterd_restart_rebalance (glusterd_conf_t *conf)
+{
+        glusterd_volinfo_t       *volinfo = NULL;
+        int                      ret = 0;
+        char                     op_errstr[256];
+
+        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                if (!volinfo->rebal.defrag_cmd)
+                        continue;
+                glusterd_volume_defrag_restart (volinfo, op_errstr, 256,
+                                        volinfo->rebal.defrag_cmd, NULL);
+        }
+        return ret;
+}
+
+void
+glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo)
+{
+        glusterd_rebalance_t *rebal = NULL;
+        GF_ASSERT (volinfo);
+
+        rebal = &volinfo->rebal;
+        rebal->rebalance_files = 0;
+        rebal->rebalance_data = 0;
+        rebal->lookedup_files = 0;
+        rebal->rebalance_failures = 0;
+        rebal->rebalance_time = 0;
+        rebal->skipped_files = 0;
+
+}
+
+/* Return hostname for given uuid if it exists
+ * else return NULL
+ */
+char *
+glusterd_uuid_to_hostname (uuid_t uuid)
+{
+        char                    *hostname = NULL;
+        glusterd_conf_t         *priv = NULL;
+        glusterd_peerinfo_t     *entry = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (priv);
+
+        if (!uuid_compare (MY_UUID, uuid)) {
+                hostname = gf_strdup ("localhost");
+        }
+        if (!list_empty (&priv->peers)) {
+                list_for_each_entry (entry, &priv->peers, uuid_list) {
+                        if (!uuid_compare (entry->uuid, uuid)) {
+                                hostname = gf_strdup (entry->hostname);
+                                break;
+                        }
+                }
+        }
+
+        return hostname;
+}
+
+gf_boolean_t
+glusterd_is_local_brick (xlator_t *this, glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo)
+{
+        gf_boolean_t    local = _gf_false;
+        int             ret = 0;
+        glusterd_conf_t *conf = NULL;
+
+        if (uuid_is_null (brickinfo->uuid)) {
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret)
+                        goto out;
+        }
+        conf = this->private;
+        local = !uuid_compare (brickinfo->uuid, MY_UUID);
+out:
+        return local;
+}
+int
+glusterd_validate_volume_id (dict_t *op_dict, glusterd_volinfo_t *volinfo)
+{
+        int     ret             = -1;
+        char    *volid_str      = NULL;
+        uuid_t  vol_uid         = {0, };
+        xlator_t *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (op_dict, "vol-id", &volid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume id for "
+                        "volume %s", volinfo->volname);
+                goto out;
+        }
+        ret = uuid_parse (volid_str, vol_uid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to parse volume id "
+                        "for volume %s", volinfo->volname);
+                goto out;
+        }
+
+        if (uuid_compare (vol_uid, volinfo->volume_id)) {
+                gf_log (this->name, GF_LOG_ERROR, "Volume ids of volume %s - %s"
+                        " and %s - are different. Possibly a split brain among "
+                        "peers.", volinfo->volname, volid_str,
+                        uuid_utoa (volinfo->volume_id));
+                ret = -1;
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
+                                      dict_t *rsp_dict)
+{
+        int                             ret = 0;
+        uint64_t                        files = 0;
+        uint64_t                        size = 0;
+        uint64_t                        lookup = 0;
+        gf_defrag_status_t              status = GF_DEFRAG_STATUS_NOT_STARTED;
+        uint64_t                        failures = 0;
+        uint64_t                        skipped = 0;
+        xlator_t                       *this = NULL;
+        double                          run_time = 0;
+
+        this = THIS;
+
+        ret = dict_get_uint64 (rsp_dict, "files", &files);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get file count");
+
+        ret = dict_get_uint64 (rsp_dict, "size", &size);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get size of xfer");
+
+        ret = dict_get_uint64 (rsp_dict, "lookups", &lookup);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get lookedup file count");
+
+        ret = dict_get_int32 (rsp_dict, "status", (int32_t *)&status);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get status");
+
+        ret = dict_get_uint64 (rsp_dict, "failures", &failures);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get failure count");
+
+        ret = dict_get_uint64 (rsp_dict, "skipped", &skipped);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get skipped count");
+
+        ret = dict_get_double (rsp_dict, "run-time", &run_time);
+        if (ret)
+                gf_log (this->name, GF_LOG_TRACE,
+                        "failed to get run-time");
+
+        if (files)
+                volinfo->rebal.rebalance_files = files;
+        if (size)
+                volinfo->rebal.rebalance_data = size;
+        if (lookup)
+                volinfo->rebal.lookedup_files = lookup;
+        if (status)
+                volinfo->rebal.defrag_status = status;
+        if (failures)
+                volinfo->rebal.rebalance_failures = failures;
+        if (skipped)
+                volinfo->rebal.skipped_files = skipped;
+        if (run_time)
+                volinfo->rebal.rebalance_time = run_time;
+
+        return ret;
+}
+
+int
+glusterd_check_topology_identical (const char *filename1,
+                                   const char *filename2,
+                                   gf_boolean_t *identical)
+{
+        int                     ret = -1; /* FAILURE */
+        xlator_t                *this = NULL;
+        FILE                    *fp1 = NULL;
+        FILE                    *fp2 = NULL;
+        glusterfs_graph_t       *grph1 = NULL;
+        glusterfs_graph_t       *grph2 = NULL;
+
+        if ((!filename1) || (!filename2) || (!identical))
+                goto out;
+
+        this = THIS;
+
+        errno = 0; /* RESET the errno */
+
+        /* fopen() the volfile1 to create the graph */
+        fp1 = fopen (filename1, "r");
+        if (fp1 == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
+                        "(%s)", filename1, strerror (errno));
+                goto out;
+        }
+
+        /* fopen() the volfile2 to create the graph */
+        fp2 = fopen (filename2, "r");
+        if (fp2 == NULL) {
+                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
+                        "(%s)", filename2, strerror (errno));
+                goto out;
+        }
+
+        /* create the graph for filename1 */
+        grph1 = glusterfs_graph_construct(fp1);
+        if (grph1 == NULL)
+                goto out;
+
+        /* create the graph for filename2 */
+        grph2 = glusterfs_graph_construct(fp2);
+        if (grph2 == NULL)
+                goto out;
+
+        /* compare the graph topology */
+        *identical = is_graph_topology_equal(grph1, grph2);
+        ret = 0; /* SUCCESS */
+out:
+        if (fp1)
+                fclose(fp1);
+        if (fp2)
+                fclose(fp2);
+        if (grph1)
+                glusterfs_graph_destroy(grph1);
+        if (grph2)
+                glusterfs_graph_destroy(grph2);
+
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
+glusterd_check_files_identical (char *filename1, char *filename2,
+                                gf_boolean_t *identical)
+{
+        int                     ret = -1;
+        struct stat             buf1 = {0,};
+        struct stat             buf2 = {0,};
+        uint32_t                cksum1 = 0;
+        uint32_t                cksum2 = 0;
+        xlator_t                *this = NULL;
+
+        GF_ASSERT (filename1);
+        GF_ASSERT (filename2);
+        GF_ASSERT (identical);
+
+        this = THIS;
+
+        ret = stat (filename1, &buf1);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "stat on file: %s failed "
+                        "(%s)", filename1, strerror (errno));
+                goto out;
+        }
+
+        ret = stat (filename2, &buf2);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "stat on file: %s failed "
+                        "(%s)", filename2, strerror (errno));
+                goto out;
+        }
+
+        if (buf1.st_size != buf2.st_size) {
+                *identical = _gf_false;
+                goto out;
+        }
+
+        ret = get_checksum_for_path (filename1, &cksum1);
+        if (ret)
+                goto out;
+
+
+        ret = get_checksum_for_path (filename2, &cksum2);
+        if (ret)
+                goto out;
+
+        if (cksum1 != cksum2)
+                *identical = _gf_false;
+        else
+                *identical = _gf_true;
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        return ret;
+}
+
+int
+glusterd_volset_help (dict_t *dict, char **op_errstr)
+{
+        int                     ret = -1;
+        gf_boolean_t            xml_out = _gf_false;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+
+        if (!dict) {
+                if (!(dict = glusterd_op_get_ctx ())) {
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+        if (dict_get (dict, "help" )) {
+                xml_out = _gf_false;
+
+        } else if (dict_get (dict, "help-xml" )) {
+                xml_out = _gf_true;
+#if (HAVE_LIB_XML)
+                ret = 0;
+#else
+                gf_log (this->name, GF_LOG_ERROR,
+                        "libxml not present in the system");
+                if (op_errstr)
+                        *op_errstr = gf_strdup ("Error: xml libraries not "
+                                                "present to produce "
+                                                "xml-output");
+                goto out;
+#endif
+
+        } else {
+                goto out;
+        }
+
+        ret = glusterd_get_volopt_content (dict, xml_out);
+        if (ret && op_errstr)
+                *op_errstr = gf_strdup ("Failed to get volume options help");
+ out:
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_to_cli (rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
+                 int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
+                 dict_t *dict)
+{
+        int                ret = -1;
+        char               *cmd = NULL;
+        int                op_ret = 0;
+        char               *op_errstr = NULL;
+        int                op_errno = 0;
+        xlator_t           *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        op_ret = arg->op_ret;
+        op_errstr = arg->op_errstr;
+        op_errno = arg->op_errno;
+
+        ret = dict_get_str (dict, "cmd-str", &cmd);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get command "
+                        "string");
+
+        if (cmd) {
+                if (op_ret)
+                        gf_cmd_log ("", "%s : FAILED %s %s", cmd,
+                                       (op_errstr)? ":" : " ",
+                                       (op_errstr)? op_errstr : " ");
+                else
+                        gf_cmd_log ("", "%s : SUCCESS", cmd);
+        }
+
+        glusterd_submit_reply (req, arg, payload, payloadcount, iobref,
+                               (xdrproc_t) xdrproc);
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+static int32_t
+glusterd_append_gsync_status (dict_t *dst, dict_t *src)
+{
+        int                ret = 0;
+        char               *stop_msg = NULL;
+
+        ret = dict_get_str (src, "gsync-status", &stop_msg);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dst, "gsync-status", gf_strdup (stop_msg));
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_WARNING, "Unable to set the stop"
+                        "message in the ctx dictionary");
+                goto out;
+        }
+
+        ret = 0;
+ out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int32_t
+glusterd_append_status_dicts (dict_t *dst, dict_t *src)
+{
+        int              dst_count = 0;
+        int              src_count = 0;
+        int              i = 0;
+        int              ret = 0;
+        char             mst[PATH_MAX] = {0,};
+        char             slv[PATH_MAX] = {0, };
+        char             sts[PATH_MAX] = {0, };
+        char             nds[PATH_MAX] = {0, };
+        char             *mst_val = NULL;
+        char             *slv_val = NULL;
+        char             *sts_val = NULL;
+        char             *nds_val = NULL;
+
+        GF_ASSERT (dst);
+
+        if (src == NULL)
+                goto out;
+
+        ret = dict_get_int32 (dst, "gsync-count", &dst_count);
+        if (ret)
+                dst_count = 0;
+
+        ret = dict_get_int32 (src, "gsync-count", &src_count);
+        if (ret || !src_count) {
+                gf_log ("", GF_LOG_DEBUG, "Source brick empty");
+                ret = 0;
+                goto out;
+        }
+
+        for (i = 1; i <= src_count; i++) {
+                snprintf (nds, sizeof(nds), "node%d", i);
+                snprintf (mst, sizeof(mst), "master%d", i);
+                snprintf (slv, sizeof(slv), "slave%d", i);
+                snprintf (sts, sizeof(sts), "status%d", i);
+
+                ret = dict_get_str (src, nds, &nds_val);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (src, mst, &mst_val);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (src, slv, &slv_val);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (src, sts, &sts_val);
+                if (ret)
+                        goto out;
+
+                snprintf (nds, sizeof(nds), "node%d", i+dst_count);
+                snprintf (mst, sizeof(mst), "master%d", i+dst_count);
+                snprintf (slv, sizeof(slv), "slave%d", i+dst_count);
+                snprintf (sts, sizeof(sts), "status%d", i+dst_count);
+
+                ret = dict_set_dynstr (dst, nds, gf_strdup (nds_val));
+                if (ret)
+                        goto out;
+
+                ret = dict_set_dynstr (dst, mst, gf_strdup (mst_val));
+                if (ret)
+                        goto out;
+
+                ret = dict_set_dynstr (dst, slv, gf_strdup (slv_val));
+                if (ret)
+                        goto out;
+
+                ret = dict_set_dynstr (dst, sts, gf_strdup (sts_val));
+                if (ret)
+                        goto out;
+
+        }
+
+        ret = dict_set_int32 (dst, "gsync-count", dst_count+src_count);
+
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+int32_t
+glusterd_gsync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict, char *op_errstr)
+{
+        dict_t             *ctx = NULL;
+        int                ret = 0;
+        char               *conf_path = NULL;
+
+        if (aggr) {
+                ctx = aggr;
+
+        } else {
+                ctx = glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Operation Context is not present");
+                        GF_ASSERT (0);
+                }
+        }
+
+        if (rsp_dict) {
+                ret = glusterd_append_status_dicts (ctx, rsp_dict);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_append_gsync_status (ctx, rsp_dict);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_str (rsp_dict, "conf_path", &conf_path);
+                if (!ret && conf_path) {
+                        ret = dict_set_dynstr (ctx, "conf_path",
+                                            gf_strdup(conf_path));
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to store conf path.");
+                                goto out;
+                        }
+                }
+        }
+        if ((op_errstr) && (strcmp ("", op_errstr))) {
+                ret = dict_set_dynstr (ctx, "errstr", gf_strdup(op_errstr));
+                if (ret)
+                        goto out;
+        }
+
+        ret = 0;
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
+        return ret;
+}
+
+int32_t
+glusterd_rb_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int32_t  src_port = 0;
+        int32_t  dst_port = 0;
+        int      ret      = 0;
+        dict_t  *ctx      = NULL;
+
+
+        if (aggr) {
+                ctx = aggr;
+
+        } else {
+                ctx = glusterd_op_get_ctx ();
+                if (!ctx) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Operation Context is not present");
+                        GF_ASSERT (0);
+                }
+        }
+
+        if (rsp_dict) {
+                ret = dict_get_int32 (rsp_dict, "src-brick-port", &src_port);
+                if (ret == 0) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "src-brick-port=%d found", src_port);
+                }
+
+                ret = dict_get_int32 (rsp_dict, "dst-brick-port", &dst_port);
+                if (ret == 0) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "dst-brick-port=%d found", dst_port);
+                }
+
+        }
+
+        if (src_port) {
+                ret = dict_set_int32 (ctx, "src-brick-port",
+                                      src_port);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not set src-brick");
+                        goto out;
+                }
+        }
+
+        if (dst_port) {
+                ret = dict_set_int32 (ctx, "dst-brick-port",
+                                      dst_port);
+                if (ret) {
+                        gf_log ("", GF_LOG_DEBUG,
+                                "Could not set dst-brick");
+                        goto out;
+                }
+
+        }
+
+out:
+        return ret;
+
+}
+
+int32_t
+glusterd_sync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int      ret      = 0;
+
+        GF_ASSERT (rsp_dict);
+
+        if (!rsp_dict) {
+                goto out;
+        }
+
+        ret = glusterd_import_friend_volumes (rsp_dict);
+out:
+        return ret;
+
+}
+
+static int
+_profile_volume_add_friend_rsp (dict_t *this, char *key, data_t *value,
+                               void *data)
+{
+        char    new_key[256] = {0};
+        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
+        data_t  *new_value = NULL;
+        int     brick_count = 0;
+        char    brick_key[256];
+
+        if (strcmp (key, "count") == 0)
+                return 0;
+        sscanf (key, "%d%s", &brick_count, brick_key);
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+        snprintf (new_key, sizeof (new_key), "%d%s",
+                  rsp_ctx->count + brick_count, brick_key);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+        return 0;
+}
+
+int
+glusterd_profile_volume_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int     ret = 0;
+        glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
+        int32_t brick_count = 0;
+        int32_t count = 0;
+        dict_t  *ctx_dict = NULL;
+        glusterd_op_t   op = GD_OP_NONE;
+
+        GF_ASSERT (rsp_dict);
+
+        ret = dict_get_int32 (rsp_dict, "count", &brick_count);
+        if (ret) {
+                ret = 0; //no bricks in the rsp
+                goto out;
+        }
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (GD_OP_PROFILE_VOLUME == op);
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx ();
+        }
+
+        ret = dict_get_int32 (ctx_dict, "count", &count);
+        rsp_ctx.count = count;
+        rsp_ctx.dict = ctx_dict;
+        dict_foreach (rsp_dict, _profile_volume_add_friend_rsp, &rsp_ctx);
+        dict_del (ctx_dict, "count");
+        ret = dict_set_int32 (ctx_dict, "count", count + brick_count);
+out:
+        return ret;
+}
+
+static int
+glusterd_volume_status_add_peer_rsp (dict_t *this, char *key, data_t *value,
+                                     void *data)
+{
+        glusterd_status_rsp_conv_t      *rsp_ctx = NULL;
+        data_t                          *new_value = NULL;
+        char                            brick_key[1024] = {0,};
+        char                            new_key[1024] = {0,};
+        int32_t                         index = 0;
+        int32_t                         ret = 0;
+
+        /* Skip the following keys, they are already present in the ctx_dict */
+        if (!strcmp (key, "count") || !strcmp (key, "cmd") ||
+            !strcmp (key, "brick-index-max") || !strcmp (key, "other-count"))
+                return 0;
+
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+
+        sscanf (key, "brick%d.%s", &index, brick_key);
+
+        if (index > rsp_ctx->brick_index_max) {
+                snprintf (new_key, sizeof (new_key), "brick%d.%s",
+                          index + rsp_ctx->other_count, brick_key);
+        } else {
+                strncpy (new_key, key, sizeof (new_key));
+                new_key[sizeof (new_key) - 1] = 0;
+        }
+
+        ret = dict_set (rsp_ctx->dict, new_key, new_value);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "Unable to set key: %s in dict",
+                        key);
+
+        return 0;
+}
+
+int
+glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int                             ret = 0;
+        glusterd_status_rsp_conv_t      rsp_ctx = {0};
+        int32_t                         cmd = GF_CLI_STATUS_NONE;
+        int32_t                         node_count = 0;
+        int32_t                         other_count = 0;
+        int32_t                         brick_index_max = -1;
+        int32_t                         rsp_node_count = 0;
+        int32_t                         rsp_other_count = 0;
+        int                             vol_count = -1;
+        int                             i = 0;
+        dict_t                          *ctx_dict = NULL;
+        char                            key[PATH_MAX] = {0,};
+        char                            *volname = NULL;
+
+        GF_ASSERT (rsp_dict);
+
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx (GD_OP_STATUS_VOLUME);
+
+        }
+
+        ret = dict_get_int32 (ctx_dict, "cmd", &cmd);
+        if (ret)
+                goto out;
+
+        if (cmd & GF_CLI_STATUS_ALL && is_origin_glusterd (ctx_dict)) {
+                ret = dict_get_int32 (rsp_dict, "vol_count", &vol_count);
+                if (ret == 0) {
+                        ret = dict_set_int32 (ctx_dict, "vol_count",
+                                              vol_count);
+                        if (ret)
+                                goto out;
+
+                        for (i = 0; i < vol_count; i++) {
+                                memset (key, 0, sizeof (key));
+                                snprintf (key, sizeof (key), "vol%d", i);
+                                ret = dict_get_str (rsp_dict, key, &volname);
+                                if (ret)
+                                        goto out;
+
+                                ret = dict_set_str (ctx_dict, key, volname);
+                                if (ret)
+                                        goto out;
+                        }
+                }
+        }
+
+        if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
+                dict_copy (rsp_dict, aggr);
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_int32 (rsp_dict, "count", &rsp_node_count);
+        if (ret) {
+                ret = 0; //no bricks in the rsp
+                goto out;
+        }
+
+        ret = dict_get_int32 (rsp_dict, "other-count", &rsp_other_count);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to get other count from rsp_dict");
+                goto out;
+        }
+
+        ret = dict_get_int32 (ctx_dict, "count", &node_count);
+        ret = dict_get_int32 (ctx_dict, "other-count", &other_count);
+        if (!dict_get (ctx_dict, "brick-index-max")) {
+                ret = dict_get_int32 (rsp_dict, "brick-index-max", &brick_index_max);
+                if (ret)
+                        goto out;
+                ret = dict_set_int32 (ctx_dict, "brick-index-max", brick_index_max);
+                if (ret)
+                        goto out;
+
+        } else {
+                ret = dict_get_int32 (ctx_dict, "brick-index-max", &brick_index_max);
+        }
+
+        rsp_ctx.count = node_count;
+        rsp_ctx.brick_index_max = brick_index_max;
+        rsp_ctx.other_count = other_count;
+        rsp_ctx.dict = ctx_dict;
+
+        dict_foreach (rsp_dict, glusterd_volume_status_add_peer_rsp, &rsp_ctx);
+
+        ret = dict_set_int32 (ctx_dict, "count", node_count + rsp_node_count);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to update node count");
+                goto out;
+        }
+
+        ret = dict_set_int32 (ctx_dict, "other-count",
+                              (other_count + rsp_other_count));
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Failed to update other-count");
+out:
+        return ret;
+}
+
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        char                 key[256]      = {0,};
+        char                *node_uuid     = NULL;
+        char                *node_uuid_str = NULL;
+        char                *volname       = NULL;
+        dict_t              *ctx_dict      = NULL;
+        double               elapsed_time  = 0;
+        glusterd_conf_t     *conf          = NULL;
+        glusterd_op_t        op            = GD_OP_NONE;
+        glusterd_peerinfo_t *peerinfo      = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        int32_t              index         = 0;
+        int32_t              count         = 0;
+        int32_t              current_index = 2;
+        int32_t              value32       = 0;
+        uint64_t             value         = 0;
+        char                *peer_uuid_str = NULL;
+
+        GF_ASSERT (rsp_dict);
+        conf = THIS->private;
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT ((GD_OP_REBALANCE == op) ||
+                   (GD_OP_DEFRAG_BRICK_VOLUME == op));
+
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx (op);
+
+        }
+
+        if (!ctx_dict)
+                goto out;
+
+        ret = dict_get_str (ctx_dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (rsp_dict, "count", &index);
+        if (ret)
+                gf_log ("", GF_LOG_ERROR, "failed to get index");
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "node-uuid-%d", index);
+        ret = dict_get_str (rsp_dict, key, &node_uuid);
+        if (!ret) {
+                node_uuid_str = gf_strdup (node_uuid);
+
+                /* Finding the index of the node-uuid in the peer-list */
+                list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
+                        peer_uuid_str = gd_peer_uuid_str (peerinfo);
+                        if (strcmp (peer_uuid_str, node_uuid_str) == 0)
+                                break;
+
+                        current_index++;
+                }
+
+                /* Setting the largest index value as the total count. */
+                ret = dict_get_int32 (ctx_dict, "count", &count);
+                if (count < current_index) {
+                    ret = dict_set_int32 (ctx_dict, "count", current_index);
+                    if (ret)
+                            gf_log ("", GF_LOG_ERROR, "Failed to set count");
+                }
+
+                /* Setting the same index for the node, as is in the peerlist.*/
+                memset (key, 0, 256);
+                snprintf (key, 256, "node-uuid-%d", current_index);
+                ret = dict_set_dynstr (ctx_dict, key, node_uuid_str);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set node-uuid");
+                }
+        }
+
+        snprintf (key, 256, "files-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "files-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set the file count");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "size-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "size-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set the size of migration");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "lookups-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "lookups-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set lookuped file count");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "status-%d", index);
+        ret = dict_get_int32 (rsp_dict, key, &value32);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "status-%d", current_index);
+                ret = dict_set_int32 (ctx_dict, key, value32);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set status");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "failures-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "failures-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set failure count");
+                }
+        }
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "skipped-%d", index);
+        ret = dict_get_uint64 (rsp_dict, key, &value);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "skipped-%d", current_index);
+                ret = dict_set_uint64 (ctx_dict, key, value);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set skipped count");
+                }
+        }
+        memset (key, 0, 256);
+        snprintf (key, 256, "run-time-%d", index);
+        ret = dict_get_double (rsp_dict, key, &elapsed_time);
+        if (!ret) {
+                memset (key, 0, 256);
+                snprintf (key, 256, "run-time-%d", current_index);
+                ret = dict_set_double (ctx_dict, key, elapsed_time);
+                if (ret) {
+                        gf_log (THIS->name, GF_LOG_DEBUG,
+                                "failed to set run-time");
+                }
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int
+glusterd_snap_config_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char           buf[PATH_MAX]        = "";
+        char          *volname              = NULL;
+        int            ret                  = -1;
+        int            config_command       = 0;
+        uint64_t       i                    = 0;
+        uint64_t       value                = 0;
+        uint64_t       voldisplaycount      = 0;
+
+        if (!dst || !src) {
+                gf_log ("", GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "config-command", &config_command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR,
+                        "failed to get config-command type");
+                goto out;
+        }
+
+        switch (config_command) {
+        case GF_SNAP_CONFIG_DISPLAY:
+                ret = dict_get_uint64 (src, "snap-max-hard-limit", &value);
+                if (!ret) {
+                        ret = dict_set_uint64 (dst, "snap-max-hard-limit", value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set snap_max_hard_limit");
+                                goto out;
+                        }
+                } else {
+                        /* Received dummy response from other nodes */
+                        ret = 0;
+                        goto out;
+                }
+
+                ret = dict_get_uint64 (src, "snap-max-soft-limit", &value);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to get snap_max_soft_limit");
+                        goto out;
+                }
+
+                ret = dict_set_uint64 (dst, "snap-max-soft-limit", value);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set snap_max_soft_limit");
+                        goto out;
+                }
+
+                ret = dict_get_uint64 (src, "voldisplaycount",
+                                       &voldisplaycount);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to get voldisplaycount");
+                        goto out;
+                }
+
+                ret = dict_set_uint64 (dst, "voldisplaycount",
+                                       voldisplaycount);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "Unable to set voldisplaycount");
+                        goto out;
+                }
+
+                for (i = 0; i < voldisplaycount; i++) {
+                        snprintf (buf, sizeof(buf), "volume%ld-volname", i);
+                        ret = dict_get_str (src, buf, &volname);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_str (dst, buf, volname);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-hard-limit", i);
+                        ret = dict_get_uint64 (src, buf, &value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_uint64 (dst, buf, value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-active-hard-limit", i);
+                        ret = dict_get_uint64 (src, buf, &value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_uint64 (dst, buf, value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+
+                        snprintf (buf, sizeof(buf),
+                                  "volume%ld-snap-max-soft-limit", i);
+                        ret = dict_get_uint64 (src, buf, &value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to get %s", buf);
+                                goto out;
+                        }
+                        ret = dict_set_uint64 (dst, buf, value);
+                        if (ret) {
+                                gf_log ("", GF_LOG_ERROR,
+                                        "Unable to set %s", buf);
+                                goto out;
+                        }
+                }
+
+                break;
+        default:
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Aggregate missed_snap_counts from different nodes and save it *
+ * in the req_dict of the originator node */
+int
+glusterd_snap_create_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char          *buf                      = NULL;
+        char          *tmp_str                  = NULL;
+        char           name_buf[PATH_MAX]       = "";
+        int32_t        i                        = -1;
+        int32_t        ret                      = -1;
+        int32_t        src_missed_snap_count    = -1;
+        int32_t        dst_missed_snap_count    = -1;
+        xlator_t      *this                     = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dst || !src) {
+                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (src, "missed_snap_count",
+                              &src_missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
+                ret = 0;
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "missed_snap_count",
+                              &dst_missed_snap_count);
+        if (ret) {
+                /* Initialize dst_missed_count for the first time */
+                dst_missed_snap_count = 0;
+        }
+
+        for (i = 0; i < src_missed_snap_count; i++) {
+                 snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                           i);
+                 ret = dict_get_str (src, name_buf, &buf);
+                 if (ret) {
+                         gf_log (this->name, GF_LOG_ERROR,
+                                 "Unable to fetch %s", name_buf);
+                         goto out;
+                 }
+
+                 snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
+                           dst_missed_snap_count);
+
+                 tmp_str = gf_strdup (buf);
+                 if (!tmp_str) {
+                         ret = -1;
+                         goto out;
+                 }
+
+                 ret = dict_set_dynstr (dst, name_buf, tmp_str);
+                 if (ret) {
+                         gf_log (this->name, GF_LOG_ERROR,
+                                 "Unable to set %s", name_buf);
+                         goto out;
+                 }
+
+                 tmp_str = NULL;
+                 dst_missed_snap_count++;
+        }
+
+        ret = dict_set_int32 (dst, "missed_snap_count", dst_missed_snap_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to set dst_missed_snap_count");
+                goto out;
+        }
+
+out:
+        if (ret && tmp_str)
+                GF_FREE(tmp_str);
+
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_snap_use_rsp_dict (dict_t *dst, dict_t *src)
+{
+        int            ret            = -1;
+        int32_t        snap_command   = 0;
+
+        if (!dst || !src) {
+                gf_log ("", GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "type", &snap_command);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "unable to get the type of "
+                        "the snapshot command");
+                goto out;
+        }
+
+        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CREATE:
+        case GF_SNAP_OPTION_TYPE_DELETE:
+                ret = glusterd_snap_create_use_rsp_dict (dst, src);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to use rsp dict");
+                        goto out;
+                }
+                break;
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+                ret = glusterd_snap_config_use_rsp_dict (dst, src);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to use rsp dict");
+                        goto out;
+                }
+                break;
+        default:
+                // copy the response dictinary's contents to the dict to be
+                // sent back to the cli
+                dict_copy (src, dst);
+                break;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_sys_exec_output_rsp_dict (dict_t *dst, dict_t *src)
+{
+        char           output_name[PATH_MAX] = "";
+        char          *output = NULL;
+        int            ret      = 0;
+        int            i      = 0;
+        int            len    = 0;
+        int            src_output_count      = 0;
+        int            dst_output_count      = 0;
+
+        if (!dst || !src) {
+                gf_log ("", GF_LOG_ERROR, "Source or Destination "
+                        "dict is empty.");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dst, "output_count", &dst_output_count);
+
+        ret = dict_get_int32 (src, "output_count", &src_output_count);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "No output from source");
+                ret = 0;
+                goto out;
+        }
+
+        for (i = 1; i <= src_output_count; i++) {
+                len = snprintf (output_name, sizeof(output_name) - 1,
+                                "output_%d", i);
+                output_name[len] = '\0';
+                ret = dict_get_str (src, output_name, &output);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to fetch %s",
+                                output_name);
+                        goto out;
+                }
+
+                len = snprintf (output_name, sizeof(output_name) - 1,
+                                "output_%d", i+dst_output_count);
+                output_name[len] = '\0';
+                ret = dict_set_dynstr (dst, output_name, gf_strdup (output));
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Unable to set %s",
+                                output_name);
+                        goto out;
+                }
+        }
+
+        ret = dict_set_int32 (dst, "output_count",
+                              dst_output_count+src_output_count);
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int            ret      = 0;
+        glusterd_op_t  op       = GD_OP_NONE;
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (aggr);
+        GF_ASSERT (rsp_dict);
+
+        if (!aggr)
+                goto out;
+        dict_copy (rsp_dict, aggr);
+out:
+        return ret;
+}
+
+int
+glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+{
+        int            ret      = 0;
+        dict_t        *ctx_dict = NULL;
+        glusterd_op_t  op       = GD_OP_NONE;
+
+        GF_ASSERT (rsp_dict);
+
+        op = glusterd_op_get_op ();
+        GF_ASSERT (GD_OP_HEAL_VOLUME == op);
+
+        if (aggr) {
+                ctx_dict = aggr;
+
+        } else {
+                ctx_dict = glusterd_op_get_ctx (op);
+        }
+
+        if (!ctx_dict)
+                goto out;
+        dict_copy (rsp_dict, ctx_dict);
+out:
+        return ret;
+}
+
+int
+_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
+                             void *data)
+{
+        char    new_key[256] = {0};
+        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
+        data_t  *new_value = NULL;
+
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        GF_ASSERT (new_value);
+        snprintf (new_key, sizeof (new_key), "%d-%s", rsp_ctx->count, key);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+        return 0;
+}
+
+int
+glusterd_profile_volume_brick_rsp (void *pending_entry,
+                                   dict_t *rsp_dict, dict_t *op_ctx,
+                                   char **op_errstr, gd_node_type type)
+{
+        int                             ret = 0;
+        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
+        int32_t                         count = 0;
+        char                            brick[PATH_MAX+1024] = {0};
+        char                            key[256] = {0};
+        char                            *full_brick = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+        xlator_t                        *this = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (op_errstr);
+        GF_ASSERT (pending_entry);
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_int32 (op_ctx, "count", &count);
+        if (ret) {
+                count = 1;
+        } else {
+                count++;
+        }
+        snprintf (key, sizeof (key), "%d-brick", count);
+        if (type == GD_NODE_BRICK) {
+                brickinfo = pending_entry;
+                snprintf (brick, sizeof (brick), "%s:%s", brickinfo->hostname,
+                          brickinfo->path);
+        } else if (type == GD_NODE_NFS) {
+                snprintf (brick, sizeof (brick), "%s", uuid_utoa (MY_UUID));
+        }
+        full_brick = gf_strdup (brick);
+        GF_ASSERT (full_brick);
+        ret = dict_set_dynstr (op_ctx, key, full_brick);
+
+        rsp_ctx.count = count;
+        rsp_ctx.dict = op_ctx;
+        dict_foreach (rsp_dict, _profile_volume_add_brick_rsp, &rsp_ctx);
+        dict_del (op_ctx, "count");
+        ret = dict_set_int32 (op_ctx, "count", count);
+        return ret;
+}
+
+//input-key: <replica-id>:<child-id>-*
+//output-key: <brick-id>-*
+int
+_heal_volume_add_shd_rsp (dict_t *this, char *key, data_t *value, void *data)
+{
+        char                            new_key[256] = {0,};
+        char                            int_str[16] = {0};
+        data_t                          *new_value = NULL;
+        char                            *rxl_end = NULL;
+        char                            *rxl_child_end = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        int                             rxl_id = 0;
+        int                             rxl_child_id = 0;
+        int                             brick_id = 0;
+        int                             int_len = 0;
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+
+        rsp_ctx = data;
+        rxl_end = strchr (key, '-');
+        if (!rxl_end)
+                goto out;
+
+        int_len = strlen (key) - strlen (rxl_end);
+        strncpy (int_str, key, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_id);
+        if (ret)
+                goto out;
+
+        rxl_child_end = strchr (rxl_end + 1, '-');
+        if (!rxl_child_end)
+                goto out;
+
+        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
+        strncpy (int_str, rxl_end + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_child_id);
+        if (ret)
+                goto out;
+
+        volinfo = rsp_ctx->volinfo;
+        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
+
+        if (!strcmp (rxl_child_end, "-status")) {
+                brickinfo = glusterd_get_brickinfo_by_position (volinfo,
+                                                                brick_id);
+                if (!brickinfo)
+                        goto out;
+                if (!glusterd_is_local_brick (rsp_ctx->this, volinfo,
+                                              brickinfo))
+                        goto out;
+        }
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "%d%s", brick_id, rxl_child_end);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+out:
+        return 0;
+}
+
+int
+_heal_volume_add_shd_rsp_of_statistics (dict_t *this, char *key, data_t
+                                             *value, void *data)
+{
+        char                            new_key[256] = {0,};
+        char                            int_str[16] = {0,};
+        char                            key_begin_string[128] = {0,};
+        data_t                          *new_value = NULL;
+        char                            *rxl_end = NULL;
+        char                            *rxl_child_end = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        char                            *key_begin_str = NULL;
+        int                             rxl_id = 0;
+        int                             rxl_child_id = 0;
+        int                             brick_id = 0;
+        int                             int_len = 0;
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
+        glusterd_brickinfo_t            *brickinfo = NULL;
+
+        rsp_ctx = data;
+        key_begin_str = strchr (key, '-');
+        if (!key_begin_str)
+                goto out;
+
+        int_len = strlen (key) - strlen (key_begin_str);
+        strncpy (key_begin_string, key, int_len);
+        key_begin_string[int_len] = '\0';
+
+        rxl_end = strchr (key_begin_str + 1, '-');
+        if (!rxl_end)
+                goto out;
+
+        int_len = strlen (key_begin_str) - strlen (rxl_end) - 1;
+        strncpy (int_str, key_begin_str + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_id);
+        if (ret)
+                goto out;
+
+
+        rxl_child_end = strchr (rxl_end + 1, '-');
+        if (!rxl_child_end)
+                goto out;
+
+        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
+        strncpy (int_str, rxl_end + 1, int_len);
+        int_str[int_len] = '\0';
+        ret = gf_string2int (int_str, &rxl_child_id);
+        if (ret)
+                goto out;
+
+        volinfo = rsp_ctx->volinfo;
+        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
+
+        brickinfo = glusterd_get_brickinfo_by_position (volinfo, brick_id);
+        if (!brickinfo)
+                goto out;
+        if (!glusterd_is_local_brick (rsp_ctx->this, volinfo, brickinfo))
+                goto out;
+
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "%s-%d%s", key_begin_string,
+                  brick_id, rxl_child_end);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+out:
+        return 0;
+
+}
+
+int
+glusterd_heal_volume_brick_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                dict_t *op_ctx, char **op_errstr)
+{
+        int                             ret = 0;
+        glusterd_heal_rsp_conv_t        rsp_ctx = {0};
+        char                            *volname = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        int                             heal_op = -1;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_str (req_dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = dict_get_int32 (req_dict, "heal-op", &heal_op);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get heal_op");
+                goto out;
+        }
+
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret)
+                goto out;
+
+        rsp_ctx.dict = op_ctx;
+        rsp_ctx.volinfo = volinfo;
+        rsp_ctx.this = THIS;
+        if (heal_op == GF_AFR_OP_STATISTICS)
+                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp_of_statistics,
+                              &rsp_ctx);
+        else
+                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp, &rsp_ctx);
+
+
+out:
+        return ret;
+}
+
+int
+_status_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
+                              void *data)
+{
+        char                            new_key[256] = {0,};
+        data_t                          *new_value = 0;
+        glusterd_pr_brick_rsp_conv_t    *rsp_ctx = NULL;
+
+        rsp_ctx = data;
+        new_value = data_copy (value);
+        snprintf (new_key, sizeof (new_key), "brick%d.%s", rsp_ctx->count, key);
+        dict_set (rsp_ctx->dict, new_key, new_value);
+
+        return 0;
+}
+
+int
+glusterd_status_volume_brick_rsp (dict_t *rsp_dict, dict_t *op_ctx,
+                                  char **op_errstr)
+{
+        int                             ret = 0;
+        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
+        int32_t                         count = 0;
+        int                             index = 0;
+
+        GF_ASSERT (rsp_dict);
+        GF_ASSERT (op_ctx);
+        GF_ASSERT (op_errstr);
+
+        ret = dict_get_int32 (op_ctx, "count", &count);
+        if (ret) {
+                count = 0;
+        } else {
+                count++;
+        }
+        ret = dict_get_int32 (rsp_dict, "index", &index);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get node index");
+                goto out;
+        }
+        dict_del (rsp_dict, "index");
+
+        rsp_ctx.count = index;
+        rsp_ctx.dict = op_ctx;
+        dict_foreach (rsp_dict, _status_volume_add_brick_rsp, &rsp_ctx);
+        ret = dict_set_int32 (op_ctx, "count", count);
+
+out:
+        return ret;
+}
+
+int
+glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
+                                 dict_t *op_ctx)
+{
+        int                             ret = 0;
+        char                            *volname = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        char                            key[256] = {0,};
+        int32_t                         i = 0;
+        char                            buf[1024] = {0,};
+        char                            *node_str = NULL;
+        glusterd_conf_t                 *priv = NULL;
+
+        priv = THIS->private;
+        GF_ASSERT (req_dict);
+
+        ret = dict_get_str (req_dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+
+        if (ret)
+                goto out;
+
+        if (rsp_dict) {
+                ret = glusterd_defrag_volume_status_update (volinfo,
+                                                            rsp_dict);
+        }
+
+        if (!op_ctx) {
+                dict_copy (rsp_dict, op_ctx);
+                goto out;
+        }
+
+        ret = dict_get_int32 (op_ctx, "count", &i);
+        i++;
+
+        ret = dict_set_int32 (op_ctx, "count", i);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set count");
+
+        snprintf (buf, 1024, "%s", uuid_utoa (MY_UUID));
+        node_str = gf_strdup (buf);
+
+        snprintf (key, 256, "node-uuid-%d",i);
+        ret = dict_set_dynstr (op_ctx, key, node_str);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set node-uuid");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "files-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set file count");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "size-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_data);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set size of xfer");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "lookups-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.lookedup_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set lookedup file count");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "status-%d", i);
+        ret = dict_set_int32 (op_ctx, key, volinfo->rebal.defrag_status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set status");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "failures-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set failure count");
+
+        memset (key, 0 , 256);
+        snprintf (key, 256, "skipped-%d", i);
+        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.skipped_files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set skipped count");
+
+        memset (key, 0, 256);
+        snprintf (key, 256, "run-time-%d", i);
+        ret = dict_set_double (op_ctx, key, volinfo->rebal.rebalance_time);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "failed to set run-time");
+
+out:
+        return ret;
+}
+int32_t
+glusterd_handle_node_rsp (dict_t *req_dict, void *pending_entry,
+                          glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
+                          char **op_errstr, gd_node_type type)
+{
+        int                     ret = 0;
+
+        GF_ASSERT (op_errstr);
+
+        switch (op) {
+        case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_profile_volume_brick_rsp (pending_entry,
+                                                         rsp_dict, op_ctx,
+                                                         op_errstr, type);
+                break;
+        case GD_OP_STATUS_VOLUME:
+                ret = glusterd_status_volume_brick_rsp (rsp_dict, op_ctx,
+                                                        op_errstr);
+                break;
+
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+                glusterd_defrag_volume_node_rsp (req_dict,
+                                                 rsp_dict, op_ctx);
+                break;
+
+        case GD_OP_HEAL_VOLUME:
+                ret = glusterd_heal_volume_brick_rsp (req_dict, rsp_dict,
+                                                      op_ctx, op_errstr);
+                break;
+        default:
+                break;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+/* Should be used only when an operation is in progress, as that is the only
+ * time a lock_owner is set
+ */
+gf_boolean_t
+is_origin_glusterd (dict_t *dict)
+{
+        gf_boolean_t  ret              = _gf_false;
+        uuid_t        lock_owner       = {0,};
+        uuid_t        *originator_uuid = NULL;
+
+        GF_ASSERT (dict);
+
+        ret = dict_get_bin (dict, "originator_uuid",
+                            (void **) &originator_uuid);
+        if (ret) {
+                /* If not originator_uuid has been set, then the command
+                 * has been originated from a glusterd running on older version
+                 * Hence fetching the lock owner */
+                ret = glusterd_get_lock_owner (&lock_owner);
+                if (ret) {
+                        ret = _gf_false;
+                        goto out;
+                }
+                ret = !uuid_compare (MY_UUID, lock_owner);
+        } else
+                ret = !uuid_compare (MY_UUID, *originator_uuid);
+
+out:
+        return ret;
+}
+
+int
+glusterd_generate_and_set_task_id (dict_t *dict, char *key)
+{
+        int             ret = -1;
+        uuid_t          task_id = {0,};
+        char            *uuid_str = NULL;
+        xlator_t        *this = NULL;
+
+        GF_ASSERT (dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        uuid_generate (task_id);
+        uuid_str = gf_strdup (uuid_utoa (task_id));
+        if (!uuid_str) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (dict, key, uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s in dict",
+                        key);
+                goto out;
+        }
+        gf_log (this->name, GF_LOG_INFO, "Generated task-id %s for key %s",
+                uuid_str, key);
+
+out:
+        if (ret)
+                GF_FREE (uuid_str);
+        return ret;
+}
+
+int
+glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key)
+{
+        int             ret = -1;
+        char            tmp_str[40] = {0,};
+        char            *task_id_str = NULL;
+
+        GF_ASSERT (dict);
+        GF_ASSERT (key);
+
+        uuid_unparse (uuid, tmp_str);
+        task_id_str = gf_strdup (tmp_str);
+        if (!task_id_str)
+                return -1;
+
+        ret = dict_set_dynstr (dict, key, task_id_str);
+        if (ret) {
+                GF_FREE (task_id_str);
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "Error setting uuid in dict with key %s", key);
+        }
+
+        return 0;
+}
+
+int
+_update_volume_op_versions (dict_t *this, char *key, data_t *value, void *data)
+{
+        int                op_version = 0;
+        glusterd_volinfo_t *ctx       = NULL;
+        gf_boolean_t       enabled    = _gf_true;
+        int                ret        = -1;
+
+        GF_ASSERT (data);
+        ctx = data;
+
+        op_version = glusterd_get_op_version_for_key (key);
+
+        if (gd_is_xlator_option (key) || gd_is_boolean_option (key)) {
+                ret = gf_string2boolean (value->data, &enabled);
+                if (ret)
+                        return 0;
+
+                if (!enabled)
+                        return 0;
+        }
+
+        if (op_version > ctx->op_version)
+                ctx->op_version = op_version;
+
+        if (gd_is_client_option (key) &&
+            (op_version > ctx->client_op_version))
+                ctx->client_op_version = op_version;
+
+        return 0;
+}
+
+void
+gd_update_volume_op_versions (glusterd_volinfo_t *volinfo)
+{
+        glusterd_conf_t *conf = NULL;
+        gf_boolean_t    ob_enabled = _gf_false;
+
+        GF_ASSERT (volinfo);
+
+        conf = THIS->private;
+        GF_ASSERT (conf);
+
+        /* Reset op-versions to minimum */
+        volinfo->op_version = 1;
+        volinfo->client_op_version = 1;
+
+        dict_foreach (volinfo->dict, _update_volume_op_versions, volinfo);
+
+        /* Special case for open-behind
+         * If cluster op-version >= 2 and open-behind hasn't been explicitly
+         * disabled, volume op-versions must be updated to account for it
+         */
+
+        /* TODO: Remove once we have a general way to update automatically
+         * enabled features
+         */
+        if (conf->op_version >= 2) {
+                ob_enabled = dict_get_str_boolean (volinfo->dict,
+                                                   "performance.open-behind",
+                                                   _gf_true);
+                if (ob_enabled) {
+
+                        if (volinfo->op_version < 2)
+                                volinfo->op_version = 2;
+                        if (volinfo->client_op_version < 2)
+                                volinfo->client_op_version = 2;
+                }
+        }
+
+        return;
+}
+
+/* A task is committed/completed once the task-id for it is cleared */
+gf_boolean_t
+gd_is_remove_brick_committed (glusterd_volinfo_t *volinfo)
+{
+        GF_ASSERT (volinfo);
+
+        if ((GD_OP_REMOVE_BRICK == volinfo->rebal.op) &&
+            !uuid_is_null (volinfo->rebal.rebalance_id))
+                        return _gf_false;
+
+        return _gf_true;
+}
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
+                               struct list_head *peers,
+                               char **down_peerstr)
+{
+        glusterd_peerinfo_t   *peerinfo  = NULL;
+        glusterd_brickinfo_t  *brickinfo = NULL;
+        gf_boolean_t           ret       = _gf_false;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                if (!uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                list_for_each_entry (peerinfo, peers, uuid_list) {
+                        if (uuid_compare (peerinfo->uuid, brickinfo->uuid))
+                                continue;
+
+                        /*Found peer who owns the brick, return false
+                         * if peer is not connected or not friend */
+                        if (!(peerinfo->connected) ||
+                           (peerinfo->state.state !=
+                             GD_FRIEND_STATE_BEFRIENDED)) {
+                                *down_peerstr = gf_strdup (peerinfo->hostname);
+                                gf_log ("", GF_LOG_DEBUG, "Peer %s is down. ",
+                                        peerinfo->hostname);
+                                goto out;
+                        }
+                }
+       }
+
+        ret = _gf_true;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+gf_boolean_t
+glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict)
+{
+        int           ret             = -1;
+        uint32_t      cmd             = GF_CLI_STATUS_NONE;
+        gf_boolean_t  is_status_tasks = _gf_false;
+
+        if (op != GD_OP_STATUS_VOLUME)
+                goto out;
+
+        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get opcode");
+                goto out;
+        }
+
+        if (cmd & GF_CLI_STATUS_TASKS)
+                is_status_tasks = _gf_true;
+
+out:
+        return is_status_tasks;
+}
+
+int
+glusterd_compare_snap_time(struct list_head *list1, struct list_head *list2)
+{
+        glusterd_snap_t *snap1 = NULL;
+        glusterd_snap_t *snap2 = NULL;
+        double diff_time       = 0;
+
+        GF_ASSERT (list1);
+        GF_ASSERT (list2);
+
+        snap1 = list_entry(list1, glusterd_snap_t, snap_list);
+        snap2 = list_entry(list2, glusterd_snap_t, snap_list);
+        diff_time = difftime(snap1->time_stamp, snap2->time_stamp);
+
+        return ((int)diff_time);
+}
+
+int
+glusterd_compare_snap_vol_time(struct list_head *list1, struct list_head *list2)
+{
+        glusterd_volinfo_t *snapvol1 = NULL;
+        glusterd_volinfo_t *snapvol2 = NULL;
+        double diff_time             = 0;
+
+        GF_ASSERT (list1);
+        GF_ASSERT (list2);
+
+        snapvol1 = list_entry(list1, glusterd_volinfo_t, snapvol_list);
+        snapvol2 = list_entry(list2, glusterd_volinfo_t, snapvol_list);
+        diff_time = difftime(snapvol1->snapshot->time_stamp,
+                             snapvol2->snapshot->time_stamp);
+
+        return ((int)diff_time);
+}
+
+int32_t
+glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo)
+{
+        glusterd_missed_snap_info      *new_missed_snapinfo = NULL;
+        int32_t                         ret                 = -1;
+        xlator_t                       *this                = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (missed_snapinfo);
+
+        new_missed_snapinfo = GF_CALLOC (1, sizeof(*new_missed_snapinfo),
+                                         gf_gld_mt_missed_snapinfo_t);
+
+        if (!new_missed_snapinfo)
+                goto out;
+
+        new_missed_snapinfo->node_snap_info = NULL;
+        INIT_LIST_HEAD (&new_missed_snapinfo->missed_snaps);
+        INIT_LIST_HEAD (&new_missed_snapinfo->snap_ops);
+
+        *missed_snapinfo = new_missed_snapinfo;
+
+        ret = 0;
+
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
+
+int32_t
+glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op)
+{
+        glusterd_snap_op_t      *new_snap_op = NULL;
+        int32_t                  ret         = -1;
+        xlator_t                *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (snap_op);
+
+        new_snap_op = GF_CALLOC (1, sizeof(*new_snap_op),
+                                 gf_gld_mt_missed_snapinfo_t);
+
+        if (!new_snap_op)
+                goto out;
+
+        new_snap_op->brick_path = NULL;
+        new_snap_op->brick_num = -1;
+        new_snap_op->op = -1;
+        new_snap_op->status = -1;
+        INIT_LIST_HEAD (&new_snap_op->snap_ops_list);
+
+        *snap_op = new_snap_op;
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
new file mode 100644
index 000000000..56bb799bf
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -0,0 +1,634 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_UTILS_H
+#define _GLUSTERD_UTILS_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <pthread.h>
+#include "uuid.h"
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "call-stub.h"
+#include "fd.h"
+#include "byte-order.h"
+#include "glusterd.h"
+#include "rpc-clnt.h"
+#include "protocol-common.h"
+
+#define GLUSTERD_SOCK_DIR "/var/run"
+
+struct glusterd_lock_ {
+        uuid_t  owner;
+        time_t  timestamp;
+};
+
+typedef struct glusterd_dict_ctx_ {
+        dict_t  *dict;
+        int     opt_count;
+        char    *key_name;
+        char    *val_name;
+        char    *prefix;
+} glusterd_dict_ctx_t;
+
+int
+glusterd_compare_lines (const void *a, const void *b);
+
+typedef int (*glusterd_condition_func) (glusterd_volinfo_t *volinfo,
+                                        glusterd_brickinfo_t *brickinfo,
+                                        void *ctx);
+typedef struct glusterd_lock_ glusterd_lock_t;
+
+int32_t
+glusterd_get_lock_owner (uuid_t *cur_owner);
+
+int32_t
+glusterd_lock (uuid_t new_owner);
+
+int32_t
+glusterd_unlock (uuid_t owner);
+
+int32_t
+glusterd_get_uuid (uuid_t *uuid);
+
+int
+glusterd_submit_reply (rpcsvc_request_t *req, void *arg,
+                       struct iovec *payload, int payloadcount,
+                       struct iobref *iobref, xdrproc_t xdrproc);
+
+int
+glusterd_to_cli (rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
+                 int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
+                 dict_t *dict);
+
+int
+glusterd_submit_request (struct rpc_clnt *rpc, void *req,
+                         call_frame_t *frame, rpc_clnt_prog_t *prog,
+                         int procnum, struct iobref *iobref,
+                         xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+int32_t
+glusterd_volinfo_new (glusterd_volinfo_t **volinfo);
+
+int32_t
+glusterd_volinfo_dup (glusterd_volinfo_t *volinfo,
+                      glusterd_volinfo_t **dup_volinfo,
+                      gf_boolean_t set_userauth);
+
+char *
+glusterd_auth_get_username (glusterd_volinfo_t *volinfo);
+
+char *
+glusterd_auth_get_password (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_auth_set_username (glusterd_volinfo_t *volinfo, char *username);
+
+int32_t
+glusterd_auth_set_password (glusterd_volinfo_t *volinfo, char *password);
+
+void
+glusterd_auth_cleanup (glusterd_volinfo_t *volinfo);
+
+gf_boolean_t
+glusterd_check_volume_exists (char *volname);
+
+int32_t
+glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo);
+
+int32_t
+glusterd_brickinfo_new_from_brick (char *brick, glusterd_brickinfo_t **brickinfo);
+
+int32_t
+glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo);
+
+int32_t
+glusterd_peer_destroy (glusterd_peerinfo_t *peerinfo);
+
+int32_t
+glusterd_peer_hostname_new (char *hostname, glusterd_peer_hostname_t **name);
+
+int32_t
+glusterd_snap_volinfo_find (char *volname, glusterd_snap_t *snap,
+                            glusterd_volinfo_t **volinfo);
+int32_t
+glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
+                                      glusterd_snap_t *snap,
+                                      glusterd_volinfo_t **volinfo);
+
+int32_t
+glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo);
+
+int
+glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volinfo);
+
+int
+glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
+                                         glusterd_volinfo_t **volinfo);
+
+int32_t
+glusterd_service_stop(const char *service, char *pidfile, int sig,
+                      gf_boolean_t force_kill);
+
+int32_t
+glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo);
+
+int32_t
+glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
+                                 glusterd_brickinfo_t   *brickinfo,
+                                 gf_boolean_t wait);
+
+int32_t
+glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
+                                glusterd_brickinfo_t   *brickinfo,
+                                gf_boolean_t del_brick);
+
+int32_t
+glusterd_volinfo_delete (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_brickinfo_delete (glusterd_brickinfo_t *brickinfo);
+
+gf_boolean_t
+glusterd_is_cli_op_req (int32_t op);
+
+int32_t
+glusterd_volume_brickinfo_get_by_brick (char *brick,
+                                        glusterd_volinfo_t *volinfo,
+                                        glusterd_brickinfo_t **brickinfo);
+
+int32_t
+glusterd_build_volume_dict (dict_t **vols);
+
+int32_t
+glusterd_compare_friend_data (dict_t  *vols, int32_t *status, char *hostname);
+
+int
+glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo);
+
+void
+glusterd_get_nodesvc_volfile (char *server, char *workdir,
+                                    char *volfile, size_t len);
+
+gf_boolean_t
+glusterd_is_service_running (char *pidfile, int *pid);
+
+gf_boolean_t
+glusterd_is_nodesvc_running ();
+
+gf_boolean_t
+glusterd_is_nodesvc_running ();
+
+void
+glusterd_get_nodesvc_dir (char *server, char *workdir,
+                                char *path, size_t len);
+int32_t
+glusterd_nfs_server_start ();
+
+int32_t
+glusterd_nfs_server_stop ();
+
+int32_t
+glusterd_shd_start ();
+
+int32_t
+glusterd_shd_stop ();
+
+void
+glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len);
+
+int32_t
+glusterd_nodesvc_set_socket_filepath (char *rundir, uuid_t uuid,
+                                      char *socketpath, int len);
+
+struct rpc_clnt*
+glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node);
+
+struct rpc_clnt*
+glusterd_nodesvc_get_rpc (char *server);
+
+int32_t
+glusterd_nodesvc_set_rpc (char *server, struct rpc_clnt *rpc);
+
+int32_t
+glusterd_nodesvc_connect (char *server, char *socketpath);
+
+void
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status);
+
+gf_boolean_t
+glusterd_is_nodesvc_online (char *server);
+
+int
+glusterd_remote_hostname_get (rpcsvc_request_t *req,
+                              char *remote_host, int len);
+int32_t
+glusterd_import_friend_volumes (dict_t  *vols);
+void
+glusterd_set_volume_status (glusterd_volinfo_t  *volinfo,
+                            glusterd_volume_status status);
+int
+glusterd_check_generate_start_nfs (void);
+
+int
+glusterd_check_generate_start_shd (void);
+
+int
+glusterd_nodesvcs_handle_graph_change (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_nodesvcs_handle_reconfigure (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_nodesvcs_start (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_nodesvcs_stop (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_volume_count_get (void);
+int32_t
+glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
+                             dict_t  *dict, int32_t count);
+int
+glusterd_get_brickinfo (xlator_t *this, const char *brickname,
+                        int port, gf_boolean_t localhost,
+                        glusterd_brickinfo_t **brickinfo);
+
+void
+glusterd_set_brick_status (glusterd_brickinfo_t  *brickinfo,
+                            gf_brick_status_t status);
+
+gf_boolean_t
+glusterd_is_brick_started (glusterd_brickinfo_t  *brickinfo);
+
+int
+glusterd_friend_find_by_hostname (const char *hoststr,
+                                  glusterd_peerinfo_t  **peerinfo);
+int
+glusterd_hostname_to_uuid (char *hostname, uuid_t uuid);
+
+int
+glusterd_friend_brick_belongs (glusterd_volinfo_t *volinfo,
+                               glusterd_brickinfo_t *brickinfo, void *uuid);
+int
+glusterd_all_volume_cond_check (glusterd_condition_func func, int status,
+                                void *ctx);
+int
+glusterd_brick_start (glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *brickinfo,
+                      gf_boolean_t wait);
+int
+glusterd_brick_stop (glusterd_volinfo_t *volinfo,
+                     glusterd_brickinfo_t *brickinfo,
+                     gf_boolean_t del_brick);
+
+int
+glusterd_is_defrag_on (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_volinfo_bricks_delete (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_friend_find_by_uuid (uuid_t uuid,
+                              glusterd_peerinfo_t  **peerinfo);
+int
+glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
+                             char *op_errstr, size_t len);
+int32_t
+glusterd_volume_brickinfos_delete (glusterd_volinfo_t *volinfo);
+
+int32_t
+glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
+                               glusterd_volinfo_t *volinfo,
+                               glusterd_brickinfo_t **brickinfo);
+
+int
+glusterd_brickinfo_get (uuid_t uuid, char *hostname, char *path,
+                       glusterd_brickinfo_t **brickinfo);
+int
+glusterd_is_rb_started (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_is_rb_paused (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_set_rb_status (glusterd_volinfo_t *volinfo, gf_rb_status_t status);
+
+gf_boolean_t
+glusterd_is_rb_ongoing (glusterd_volinfo_t *volinfo);
+
+int
+glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
+                          glusterd_brickinfo_t *src_brick,
+                          glusterd_brickinfo_t *dst_brick);
+
+int
+glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
+                                    char **op_errstr, gf_boolean_t is_force);
+
+int
+glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
+                                        uuid_t volume_id, char **op_errstr,
+                                        gf_boolean_t is_force);
+int
+glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
+                                           int old_state, int new_state,
+                                           int event);
+int
+glusterd_peerinfo_new (glusterd_peerinfo_t **peerinfo,
+                       glusterd_friend_sm_state_t state, uuid_t *uuid,
+                       const char *hostname, int port);
+int
+glusterd_sm_tr_log_init (glusterd_sm_tr_log_t *log,
+                         char * (*state_name_get) (int),
+                         char * (*event_name_get) (int),
+                         size_t  size);
+void
+glusterd_sm_tr_log_delete (glusterd_sm_tr_log_t *log);
+
+int
+glusterd_sm_tr_log_add_to_dict (dict_t *dict,
+                                glusterd_sm_tr_log_t *circular_log);
+int
+glusterd_remove_pending_entry (struct list_head *list, void *elem);
+int
+glusterd_clear_pending_nodes (struct list_head *list);
+gf_boolean_t
+glusterd_peerinfo_is_uuid_unknown (glusterd_peerinfo_t *peerinfo);
+int32_t
+glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
+                        glusterd_brickinfo_t  *brickinfo, char *socketpath);
+int32_t
+glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo);
+int32_t
+glusterd_delete_volume (glusterd_volinfo_t *volinfo);
+int32_t
+glusterd_delete_brick (glusterd_volinfo_t* volinfo,
+                       glusterd_brickinfo_t *brickinfo);
+
+int32_t
+glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo);
+
+int
+glusterd_spawn_daemons (void *opaque);
+
+int
+glusterd_restart_gsyncds (glusterd_conf_t *conf);
+
+int
+glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
+                      char *path_list, char *conf_path,
+                      char *glusterd_uuid_str,
+                      char **op_errstr);
+int
+glusterd_get_local_brickpaths (glusterd_volinfo_t *volinfo,
+                               char **pathlist);
+
+int32_t
+glusterd_recreate_bricks (glusterd_conf_t *conf);
+int32_t
+glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf);
+
+int
+glusterd_add_brick_detail_to_dict (glusterd_volinfo_t *volinfo,
+                                   glusterd_brickinfo_t *brickinfo,
+                                   dict_t  *dict, int32_t count);
+
+int32_t
+glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
+                            glusterd_brickinfo_t *brickinfo,
+                            dict_t  *dict, int32_t count);
+
+int32_t
+glusterd_get_all_volnames (dict_t *dict);
+
+gf_boolean_t
+glusterd_is_fuse_available ();
+
+int
+glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
+                          glusterd_brickinfo_t *brickinfo,
+                          char *options, int option_cnt, char **op_errstr);
+int
+glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr);
+gf_boolean_t
+glusterd_is_volume_replicate (glusterd_volinfo_t *volinfo);
+gf_boolean_t
+glusterd_is_brick_decommissioned (glusterd_volinfo_t *volinfo, char *hostname,
+                                  char *path);
+gf_boolean_t
+glusterd_friend_contains_vol_bricks (glusterd_volinfo_t *volinfo,
+                                     uuid_t friend_uuid);
+int
+glusterd_friend_remove_cleanup_vols (uuid_t uuid);
+
+gf_boolean_t
+glusterd_chk_peers_connected_befriended (uuid_t skip_uuid);
+
+void
+glusterd_get_client_filepath (char *filepath,
+                              glusterd_volinfo_t *volinfo,
+                              gf_transport_type type);
+void
+glusterd_get_trusted_client_filepath (char *filepath,
+                                      glusterd_volinfo_t *volinfo,
+                                      gf_transport_type type);
+int
+glusterd_restart_rebalance (glusterd_conf_t *conf);
+
+int32_t
+glusterd_add_bricks_hname_path_to_dict (dict_t *dict,
+                                        glusterd_volinfo_t *volinfo);
+
+int
+glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
+                           dict_t *vol_opts);
+
+char *
+glusterd_uuid_to_hostname (uuid_t uuid);
+
+int
+glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo);
+
+glusterd_brickinfo_t*
+glusterd_get_brickinfo_by_position (glusterd_volinfo_t *volinfo, uint32_t pos);
+
+gf_boolean_t
+glusterd_is_local_brick (xlator_t *this, glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo);
+int
+glusterd_validate_volume_id (dict_t *op_dict, glusterd_volinfo_t *volinfo);
+
+int
+glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
+                                      dict_t *rsp_dict);
+
+int
+glusterd_check_files_identical (char *filename1, char *filename2,
+                                gf_boolean_t *identical);
+
+int
+glusterd_check_topology_identical (const char *filename1,
+                                   const char *filename2,
+                                   gf_boolean_t *identical);
+
+void
+glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo);
+int
+glusterd_volset_help (dict_t *dict, char **op_errstr);
+
+int32_t
+glusterd_sync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int32_t
+glusterd_gsync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict, char *op_errstr);
+int32_t
+glusterd_rb_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_profile_volume_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_sys_exec_output_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_snap_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int32_t
+glusterd_handle_node_rsp (dict_t *req_ctx, void *pending_entry,
+                          glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
+                          char **op_errstr, gd_node_type type);
+int
+glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int
+_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
+                             void *data);
+int
+glusterd_profile_volume_brick_rsp (void *pending_entry,
+                                   dict_t *rsp_dict, dict_t *op_ctx,
+                                   char **op_errstr, gd_node_type type);
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
+                               struct list_head *peers,
+                               char **down_peerstr);
+
+/* Should be used only when an operation is in progress, as that is the only
+ * time a lock_owner is set
+ */
+gf_boolean_t
+is_origin_glusterd (dict_t *dict);
+
+gf_boolean_t
+glusterd_is_quorum_changed (dict_t *options, char *option, char *value);
+
+int
+glusterd_do_quorum_action ();
+
+int
+glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
+                                    int *quorum_count);
+
+int
+glusterd_get_next_global_opt_version_str (dict_t *opts, char **version_str);
+gf_boolean_t
+glusterd_is_quorum_option (char *option);
+gf_boolean_t
+glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo);
+gf_boolean_t
+glusterd_is_any_volume_in_server_quorum (xlator_t *this);
+gf_boolean_t
+does_gd_meet_server_quorum (xlator_t *this);
+
+int
+glusterd_generate_and_set_task_id (dict_t *dict, char *key);
+
+int
+glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key);
+
+gf_boolean_t
+glusterd_is_same_address (char *name1, char *name2);
+
+void
+gd_update_volume_op_versions (glusterd_volinfo_t *volinfo);
+
+char*
+gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo);
+
+gf_boolean_t
+gd_is_remove_brick_committed (glusterd_volinfo_t *volinfo);
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
+                               struct list_head *peers,
+                               char **down_peerstr);
+
+int
+glusterd_get_slave_details_confpath (glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char **slave_ip, char **slave_vol,
+                                     char **conf_path, char **op_errstr);
+
+int
+glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
+                                      dict_t *resp_dict, char *path_list,
+                                      char *conf_path, gf_boolean_t is_force);
+
+int
+glusterd_check_gsync_running_local (char *master, char *slave,
+                                    char *conf_path,
+                                    gf_boolean_t *is_run);
+
+gf_boolean_t
+glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict);
+
+#ifdef GF_LINUX_HOST_OS
+char*
+glusterd_get_brick_mount_details (glusterd_brickinfo_t *brickinfo);
+struct mntent *
+glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab);
+int
+glusterd_get_brick_root (char *path, char **mount_point);
+#endif //LINUX_HOST
+
+int
+glusterd_compare_snap_time(struct list_head *, struct list_head *);
+
+int
+glusterd_compare_snap_vol_time(struct list_head *, struct list_head *);
+
+int32_t
+glusterd_snap_volinfo_restore (dict_t *rsp_dict,
+                               glusterd_volinfo_t *new_volinfo,
+                               glusterd_volinfo_t *snap_volinfo);
+int32_t
+glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol);
+
+int32_t
+glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo);
+
+int32_t
+glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op);
+
+int32_t
+glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict, char *snap_uuid,
+                                   glusterd_brickinfo_t *brickinfo,
+                                   int32_t brick_number, int32_t op);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
new file mode 100644
index 000000000..6f3c69e7d
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -0,0 +1,4065 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <fnmatch.h>
+#include <sys/wait.h>
+#include <dlfcn.h>
+
+#if (HAVE_LIB_XML)
+#include <libxml/encoding.h>
+#include <libxml/xmlwriter.h>
+#endif
+
+#include "xlator.h"
+#include "glusterd.h"
+#include "defaults.h"
+#include "logging.h"
+#include "dict.h"
+#include "graph-utils.h"
+#include "glusterd-store.h"
+#include "glusterd-hooks.h"
+#include "trie.h"
+#include "glusterd-mem-types.h"
+#include "cli1-xdr.h"
+#include "glusterd-volgen.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-utils.h"
+#include "run.h"
+#include "options.h"
+
+extern struct volopt_map_entry glusterd_volopt_map[];
+
+/*********************************************
+ *
+ * xlator generation / graph manipulation API
+ *
+ *********************************************/
+
+
+struct volgen_graph {
+        char **errstr;
+        glusterfs_graph_t graph;
+};
+typedef struct volgen_graph volgen_graph_t;
+
+static void
+set_graph_errstr (volgen_graph_t *graph, const char *str)
+{
+        if (!graph->errstr)
+                return;
+
+        *graph->errstr = gf_strdup (str);
+}
+
+static xlator_t *
+xlator_instantiate_va (const char *type, const char *format, va_list arg)
+{
+        xlator_t *xl = NULL;
+        char *volname = NULL;
+        int ret = 0;
+
+        ret = gf_vasprintf (&volname, format, arg);
+        if (ret < 0) {
+                volname = NULL;
+
+                goto error;
+        }
+
+        xl = GF_CALLOC (1, sizeof (*xl), gf_common_mt_xlator_t);
+        if (!xl)
+                goto error;
+        ret = xlator_set_type_virtual (xl, type);
+        if (ret)
+                goto error;
+        xl->options = get_new_dict();
+        if (!xl->options)
+                goto error;
+        xl->name = volname;
+        INIT_LIST_HEAD (&xl->volume_options);
+
+        xl->ctx = THIS->ctx;
+
+        return xl;
+
+ error:
+        gf_log ("", GF_LOG_ERROR, "creating xlator of type %s failed",
+                type);
+        GF_FREE (volname);
+        if (xl)
+                xlator_destroy (xl);
+
+        return NULL;
+}
+
+#ifdef __not_used_as_of_now_
+static xlator_t *
+xlator_instantiate (const char *type, const char *format, ...)
+{
+        va_list arg;
+        xlator_t *xl;
+
+        va_start (arg, format);
+        xl = xlator_instantiate_va (type, format, arg);
+        va_end (arg);
+
+        return xl;
+}
+#endif
+
+static int
+volgen_xlator_link (xlator_t *pxl, xlator_t *cxl)
+{
+        int ret = 0;
+
+        ret = glusterfs_xlator_link (pxl, cxl);
+        if (ret == -1) {
+                gf_log ("", GF_LOG_ERROR,
+                        "Out of memory, cannot link xlators %s <- %s",
+                        pxl->name, cxl->name);
+        }
+
+        return ret;
+}
+
+static int
+volgen_graph_link (volgen_graph_t *graph, xlator_t *xl)
+{
+        int ret = 0;
+
+        /* no need to care about graph->top here */
+        if (graph->graph.first)
+                ret = volgen_xlator_link (xl, graph->graph.first);
+        if (ret == -1) {
+                gf_log ("", GF_LOG_ERROR, "failed to add graph entry %s",
+                        xl->name);
+
+                return -1;
+        }
+
+        return 0;
+}
+
+static xlator_t *
+volgen_graph_add_as (volgen_graph_t *graph, const char *type,
+                     const char *format, ...)
+{
+        va_list arg;
+        xlator_t *xl = NULL;
+
+        va_start (arg, format);
+        xl = xlator_instantiate_va (type, format, arg);
+        va_end (arg);
+
+        if (!xl)
+                return NULL;
+
+        if (volgen_graph_link (graph, xl)) {
+                xlator_destroy (xl);
+
+                return NULL;
+        } else
+                glusterfs_graph_set_first (&graph->graph, xl);
+
+        return xl;
+}
+
+static xlator_t *
+volgen_graph_add_nolink (volgen_graph_t *graph, const char *type,
+                         const char *format, ...)
+{
+        va_list arg;
+        xlator_t *xl = NULL;
+
+        va_start (arg, format);
+        xl = xlator_instantiate_va (type, format, arg);
+        va_end (arg);
+
+        if (!xl)
+                return NULL;
+
+        glusterfs_graph_set_first (&graph->graph, xl);
+
+        return xl;
+}
+
+static xlator_t *
+volgen_graph_add (volgen_graph_t *graph, char *type, char *volname)
+{
+        char *shorttype = NULL;
+
+        shorttype = strrchr (type, '/');
+        GF_ASSERT (shorttype);
+        shorttype++;
+        GF_ASSERT (*shorttype);
+
+        return volgen_graph_add_as (graph, type, "%s-%s", volname, shorttype);
+}
+
+/* XXX Seems there is no such generic routine?
+ * Maybe should put to xlator.c ??
+ */
+static int
+xlator_set_option (xlator_t *xl, char *key, char *value)
+{
+        char *dval     = NULL;
+
+        dval = gf_strdup (value);
+        if (!dval) {
+                gf_log ("", GF_LOG_ERROR,
+                        "failed to set xlator opt: %s[%s] = %s",
+                        xl->name, key, value);
+
+                return -1;
+        }
+
+        return dict_set_dynstr (xl->options, key, dval);
+}
+
+static int
+xlator_get_option (xlator_t *xl, char *key, char **value)
+{
+        GF_ASSERT (xl);
+        return dict_get_str (xl->options, key, value);
+}
+
+static inline xlator_t *
+first_of (volgen_graph_t *graph)
+{
+        return (xlator_t *)graph->graph.first;
+}
+
+
+
+
+/**************************
+ *
+ * Trie glue
+ *
+ *************************/
+
+
+static int
+volopt_selector (int lvl, char **patt, void *param,
+                     int (*optcbk)(char *word, void *param))
+{
+        struct volopt_map_entry *vme = NULL;
+        char *w = NULL;
+        int i = 0;
+        int len = 0;
+        int ret = 0;
+        char *dot = NULL;
+
+        for (vme = glusterd_volopt_map; vme->key; vme++) {
+                w = vme->key;
+
+                for (i = 0; i < lvl; i++) {
+                        if (patt[i]) {
+                                w = strtail (w, patt[i]);
+                                GF_ASSERT (!w || *w);
+                                if (!w || *w != '.')
+                                        goto next;
+                        } else {
+                                w = strchr (w, '.');
+                                GF_ASSERT (w);
+                        }
+                        w++;
+                }
+
+                dot = strchr (w, '.');
+                if (dot) {
+                        len = dot - w;
+                        w = gf_strdup (w);
+                        if (!w)
+                                return -1;
+                        w[len] = '\0';
+                }
+                ret = optcbk (w, param);
+                if (dot)
+                        GF_FREE (w);
+                if (ret)
+                        return -1;
+ next:
+                continue;
+        }
+
+        return 0;
+}
+
+static int
+volopt_trie_cbk (char *word, void *param)
+{
+        return trie_add ((trie_t *)param, word);
+}
+
+static int
+process_nodevec (struct trienodevec *nodevec, char **hint)
+{
+        int ret = 0;
+        char *hint1 = NULL;
+        char *hint2 = NULL;
+        char *hintinfx = "";
+        trienode_t **nodes = nodevec->nodes;
+
+        if (!nodes[0]) {
+                *hint = NULL;
+                return 0;
+        }
+
+#if 0
+        /* Limit as in git */
+        if (trienode_get_dist (nodes[0]) >= 6) {
+                *hint = NULL;
+                return 0;
+        }
+#endif
+
+        if (trienode_get_word (nodes[0], &hint1))
+                return -1;
+
+        if (nodevec->cnt < 2 || !nodes[1]) {
+                *hint = hint1;
+                return 0;
+        }
+
+        if (trienode_get_word (nodes[1], &hint2))
+                return -1;
+
+        if (*hint)
+                hintinfx = *hint;
+        ret = gf_asprintf (hint, "%s or %s%s", hint1, hintinfx, hint2);
+        if (ret > 0)
+                ret = 0;
+        return ret;
+}
+
+static int
+volopt_trie_section (int lvl, char **patt, char *word, char **hint, int hints)
+{
+        trienode_t *nodes[] = { NULL, NULL };
+        struct trienodevec nodevec = { nodes, 2};
+        trie_t *trie = NULL;
+        int ret = 0;
+
+        trie = trie_new ();
+        if (!trie)
+                return -1;
+
+        if (volopt_selector (lvl, patt, trie, &volopt_trie_cbk)) {
+                trie_destroy (trie);
+
+                return -1;
+        }
+
+        GF_ASSERT (hints <= 2);
+        nodevec.cnt = hints;
+        ret = trie_measure_vec (trie, word, &nodevec);
+        if (ret || !nodevec.nodes[0])
+                trie_destroy (trie);
+
+        ret = process_nodevec (&nodevec, hint);
+        trie_destroy (trie);
+
+        return ret;
+}
+
+static int
+volopt_trie (char *key, char **hint)
+{
+        char *patt[] = { NULL };
+        char *fullhint = NULL;
+        char *dot = NULL;
+        char *dom = NULL;
+        int   len = 0;
+        int   ret = 0;
+
+        *hint = NULL;
+
+        dot = strchr (key, '.');
+        if (!dot)
+                return volopt_trie_section (1, patt, key, hint, 2);
+
+        len = dot - key;
+        dom = gf_strdup (key);
+        if (!dom)
+                return -1;
+        dom[len] = '\0';
+
+        ret = volopt_trie_section (0, NULL, dom, patt, 1);
+        GF_FREE (dom);
+        if (ret) {
+                patt[0] = NULL;
+                goto out;
+        }
+        if (!patt[0])
+                goto out;
+
+        *hint = "...";
+        ret = volopt_trie_section (1, patt, dot + 1, hint, 2);
+        if (ret)
+                goto out;
+        if (*hint) {
+                ret = gf_asprintf (&fullhint, "%s.%s", patt[0], *hint);
+                GF_FREE (*hint);
+                if (ret >= 0) {
+                        ret = 0;
+                        *hint = fullhint;
+                }
+        }
+
+ out:
+        GF_FREE (patt[0]);
+        if (ret)
+                *hint = NULL;
+
+        return ret;
+}
+
+
+
+
+/**************************
+ *
+ * Volume generation engine
+ *
+ **************************/
+
+
+typedef int (*volgen_opthandler_t) (volgen_graph_t *graph,
+                                    struct volopt_map_entry *vme,
+                                    void *param);
+
+struct opthandler_data {
+        volgen_graph_t *graph;
+        volgen_opthandler_t handler;
+        struct volopt_map_entry *vme;
+        gf_boolean_t found;
+        gf_boolean_t data_t_fake;
+        int rv;
+        char *volname;
+        void *param;
+};
+
+static int
+process_option (char *key, data_t *value, void *param)
+{
+        struct opthandler_data *odt = param;
+        struct volopt_map_entry vme = {0,};
+
+        if (odt->rv)
+                return 0;
+        odt->found = _gf_true;
+
+        vme.key = key;
+        vme.voltype = odt->vme->voltype;
+        vme.option = odt->vme->option;
+        vme.op_version = odt->vme->op_version;
+
+        if (!vme.option) {
+                vme.option = strrchr (key, '.');
+                if (vme.option)
+                        vme.option++;
+                else
+                        vme.option = key;
+        }
+        if (odt->data_t_fake)
+                vme.value = (char *)value;
+        else
+                vme.value = value->data;
+
+        odt->rv = odt->handler (odt->graph, &vme, odt->param);
+        return 0;
+}
+
+static int
+volgen_graph_set_options_generic (volgen_graph_t *graph, dict_t *dict,
+                                  void *param, volgen_opthandler_t handler)
+{
+        struct volopt_map_entry *vme = NULL;
+        struct opthandler_data odt = {0,};
+        data_t *data = NULL;
+
+        odt.graph = graph;
+        odt.handler = handler;
+        odt.param = param;
+        (void)data;
+
+        for (vme = glusterd_volopt_map; vme->key; vme++) {
+                odt.vme = vme;
+                odt.found = _gf_false;
+                odt.data_t_fake = _gf_false;
+
+                data = dict_get (dict, vme->key);
+
+                if (data)
+                        process_option (vme->key, data, &odt);
+                if (odt.rv)
+                        return odt.rv;
+
+                if (odt.found)
+                        continue;
+
+                /* check for default value */
+
+                if (vme->value) {
+                        /* stupid hack to be able to reuse dict iterator
+                         * in this context
+                         */
+                        odt.data_t_fake = _gf_true;
+                        process_option (vme->key, (data_t *)vme->value, &odt);
+                        if (odt.rv)
+                                return odt.rv;
+                }
+        }
+
+        return 0;
+}
+
+static int
+no_filter_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                          void *param)
+{
+        xlator_t *trav;
+        int ret = 0;
+
+        for (trav = first_of (graph); trav; trav = trav->next) {
+                if (strcmp (trav->type, vme->voltype) != 0)
+                        continue;
+
+                ret = xlator_set_option (trav, vme->option, vme->value);
+                if (ret)
+                        break;
+        }
+        return ret;
+}
+
+static int
+basic_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                      void *param)
+{
+        int     ret = 0;
+
+        if (vme->option[0] == '!')
+                goto out;
+
+        ret = no_filter_option_handler (graph, vme, param);
+out:
+        return ret;
+}
+
+static int
+volgen_graph_set_options (volgen_graph_t *graph, dict_t *dict)
+{
+        return volgen_graph_set_options_generic (graph, dict, NULL,
+                                                 &basic_option_handler);
+}
+
+static int
+optget_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                       void *param)
+{
+        struct volopt_map_entry *vme2 = param;
+
+        if (strcmp (vme->key, vme2->key) == 0)
+                vme2->value = vme->value;
+
+        return 0;
+}
+
+static glusterd_server_xlator_t
+get_server_xlator (char *xlator)
+{
+        glusterd_server_xlator_t subvol = GF_XLATOR_NONE;
+
+        if (strcmp (xlator, "posix") == 0)
+                subvol = GF_XLATOR_POSIX;
+        if (strcmp (xlator, "acl") == 0)
+                subvol = GF_XLATOR_ACL;
+        if (strcmp (xlator, "locks") == 0)
+                subvol = GF_XLATOR_LOCKS;
+        if (strcmp (xlator, "io-threads") == 0)
+                subvol = GF_XLATOR_IOT;
+        if (strcmp (xlator, "index") == 0)
+                subvol = GF_XLATOR_INDEX;
+        if (strcmp (xlator, "marker") == 0)
+                subvol = GF_XLATOR_MARKER;
+        if (strcmp (xlator, "io-stats") == 0)
+                subvol = GF_XLATOR_IO_STATS;
+        if (strcmp (xlator, "bd") == 0)
+                subvol = GF_XLATOR_BD;
+
+        return subvol;
+}
+
+static glusterd_client_xlator_t
+get_client_xlator (char *xlator)
+{
+        glusterd_client_xlator_t subvol = GF_CLNT_XLATOR_NONE;
+
+        if (strcmp (xlator, "client") == 0)
+                subvol = GF_CLNT_XLATOR_FUSE;
+
+        return subvol;
+}
+
+static int
+debugxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                       void *param)
+{
+        char *volname = NULL;
+        gf_boolean_t enabled = _gf_false;
+
+        volname = param;
+
+        if (strcmp (vme->option, "!debug") != 0)
+                return 0;
+
+        if (!strcmp (vme->key , "debug.trace") ||
+            !strcmp (vme->key, "debug.error-gen")) {
+                if (get_server_xlator (vme->value) == GF_XLATOR_NONE &&
+                    get_client_xlator (vme->value) == GF_CLNT_XLATOR_NONE)
+                        return 0;
+                else
+                        goto add_graph;
+        }
+
+        if (gf_string2boolean (vme->value, &enabled) == -1)
+                return -1;
+        if (!enabled)
+                return 0;
+
+add_graph:
+        if (volgen_graph_add (graph, vme->voltype, volname))
+                return 0;
+        else
+                return -1;
+}
+
+int
+check_and_add_debug_xl (volgen_graph_t *graph, dict_t *set_dict, char *volname,
+                        char *xlname)
+{
+        int       ret       = 0;
+        char     *value_str = NULL;
+
+        ret = dict_get_str (set_dict, "debug.trace", &value_str);
+        if (!ret) {
+                if (strcmp (xlname, value_str) == 0) {
+                        ret = volgen_graph_set_options_generic (graph, set_dict, volname,
+                                                                &debugxl_option_handler);
+                        if (ret)
+                                goto out;
+                }
+        }
+
+        ret = dict_get_str (set_dict, "debug.error-gen", &value_str);
+        if (!ret) {
+                if (strcmp (xlname, value_str) == 0) {
+                        ret = volgen_graph_set_options_generic (graph, set_dict, volname,
+                                                                &debugxl_option_handler);
+                        if (ret)
+                                goto out;
+                }
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+/* This getter considers defaults also. */
+static int
+volgen_dict_get (dict_t *dict, char *key, char **value)
+{
+        struct volopt_map_entry vme = {0,};
+        int ret = 0;
+
+        vme.key = key;
+
+        ret = volgen_graph_set_options_generic (NULL, dict, &vme,
+                                                &optget_option_handler);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Out of memory");
+
+                return -1;
+        }
+
+        *value = vme.value;
+
+        return 0;
+}
+
+static int
+option_complete (char *key, char **completion)
+{
+        struct volopt_map_entry *vme = NULL;
+
+        *completion = NULL;
+        for (vme = glusterd_volopt_map; vme->key; vme++) {
+                if (strcmp (strchr (vme->key, '.') + 1, key) != 0)
+                        continue;
+
+                if (*completion && strcmp (*completion, vme->key) != 0) {
+                        /* cancel on non-unique match */
+                        *completion = NULL;
+
+                        return 0;
+                } else
+                        *completion = vme->key;
+        }
+
+        if (*completion) {
+                /* For sake of unified API we want
+                 * have the completion to be a to-be-freed
+                 * string.
+                 */
+                *completion = gf_strdup (*completion);
+                return -!*completion;
+        }
+
+        return 0;
+}
+
+int
+glusterd_volinfo_get (glusterd_volinfo_t *volinfo, char *key, char **value)
+{
+        return volgen_dict_get (volinfo->dict, key, value);
+}
+
+int
+glusterd_volinfo_get_boolean (glusterd_volinfo_t *volinfo, char *key)
+{
+        char *val = NULL;
+        gf_boolean_t  boo = _gf_false;
+        int ret = 0;
+
+        ret = glusterd_volinfo_get (volinfo, key, &val);
+        if (ret)
+                return -1;
+
+        if (val)
+                ret = gf_string2boolean (val, &boo);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "value for %s option is not valid", key);
+
+                return -1;
+        }
+
+        return boo;
+}
+
+gf_boolean_t
+glusterd_check_voloption_flags (char *key, int32_t flags)
+{
+        char *completion = NULL;
+        struct volopt_map_entry *vmep = NULL;
+        int   ret = 0;
+
+        COMPLETE_OPTION(key, completion, ret);
+        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+                if (strcmp (vmep->key, key) == 0) {
+                        if (vmep->flags & flags)
+                                return _gf_true;
+                        else
+                                return _gf_false;
+                }
+        }
+
+        return _gf_false;
+}
+
+gf_boolean_t
+glusterd_check_globaloption (char *key)
+{
+        char *completion = NULL;
+        struct volopt_map_entry *vmep = NULL;
+        int   ret = 0;
+
+        COMPLETE_OPTION(key, completion, ret);
+        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+                if (strcmp (vmep->key, key) == 0) {
+                        if ((vmep->type == GLOBAL_DOC) ||
+                            (vmep->type == GLOBAL_NO_DOC))
+                                return _gf_true;
+                        else
+                                return _gf_false;
+                }
+        }
+
+        return _gf_false;
+}
+
+gf_boolean_t
+glusterd_check_localoption (char *key)
+{
+        char *completion = NULL;
+        struct volopt_map_entry *vmep = NULL;
+        int   ret = 0;
+
+        COMPLETE_OPTION(key, completion, ret);
+        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+                if (strcmp (vmep->key, key) == 0) {
+                        if ((vmep->type == DOC) ||
+                            (vmep->type == NO_DOC))
+                                return _gf_true;
+                        else
+                                return _gf_false;
+                }
+        }
+
+        return _gf_false;
+}
+
+int
+glusterd_check_voloption (char *key)
+{
+        char *completion = NULL;
+        struct volopt_map_entry *vmep = NULL;
+        int ret = 0;
+
+        COMPLETE_OPTION(key, completion, ret);
+        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+                if (strcmp (vmep->key, key) == 0) {
+                        if ((vmep->type == DOC) ||
+                            (vmep->type == DOC))
+                                return _gf_true;
+                        else
+                                return _gf_false;
+                }
+        }
+
+        return _gf_false;
+
+}
+
+int
+glusterd_check_option_exists (char *key, char **completion)
+{
+        struct volopt_map_entry vme = {0,};
+        struct volopt_map_entry *vmep = NULL;
+        int ret = 0;
+        xlator_t *this = THIS;
+
+        (void)vme;
+        (void)vmep;
+
+        if (!strchr (key, '.')) {
+                if (completion) {
+                        ret = option_complete (key, completion);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Out of memory");
+                                return -1;
+                        }
+
+                        ret = !!*completion;
+                        if (ret)
+                                return ret;
+                        else
+                                goto trie;
+                } else
+                        return 0;
+        }
+
+        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+                if (strcmp (vmep->key, key) == 0) {
+                        ret = 1;
+                        break;
+                }
+        }
+
+        if (ret || !completion)
+                return ret;
+
+ trie:
+        ret = volopt_trie (key, completion);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Some error occurred during keyword hinting");
+        }
+
+        return ret;
+}
+
+char*
+glusterd_get_trans_type_rb (gf_transport_type ttype)
+{
+        char *trans_type = NULL;
+
+        switch (ttype) {
+        case GF_TRANSPORT_RDMA:
+                gf_asprintf (&trans_type, "rdma");
+                break;
+        case GF_TRANSPORT_TCP:
+        case GF_TRANSPORT_BOTH_TCP_RDMA:
+                gf_asprintf (&trans_type, "tcp");
+                break;
+        default:
+                gf_log (THIS->name, GF_LOG_ERROR, "Unknown "
+                        "transport type");
+        }
+
+        return trans_type;
+}
+
+static int
+_xl_link_children (xlator_t *parent, xlator_t *children, size_t child_count)
+{
+        xlator_t        *trav = NULL;
+        size_t          seek = 0;
+        int             ret = -1;
+
+        if (child_count == 0)
+                goto out;
+        seek = child_count;
+        for (trav = children; --seek; trav = trav->next);
+        for (; child_count--; trav = trav->prev) {
+                ret = volgen_xlator_link (parent, trav);
+                if (ret)
+                        goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+volgen_graph_merge_sub (volgen_graph_t *dgraph, volgen_graph_t *sgraph,
+                        size_t child_count)
+{
+        xlator_t *trav = NULL;
+        int      ret   = 0;
+
+        GF_ASSERT (dgraph->graph.first);
+
+        ret = _xl_link_children (first_of (dgraph), first_of (sgraph),
+                                 child_count);
+        if (ret)
+                goto out;
+
+        for (trav = first_of (dgraph); trav->next; trav = trav->next);
+
+        trav->next = first_of (sgraph);
+        trav->next->prev = trav;
+        dgraph->graph.xl_count += sgraph->graph.xl_count;
+
+out:
+        return ret;
+}
+
+static void
+volgen_apply_filters (char *orig_volfile)
+{
+	DIR           *filterdir = NULL;
+	struct dirent  entry = {0,};
+	struct dirent *next = NULL;
+	char          *filterpath = NULL;
+	struct stat    statbuf = {0,};
+
+	filterdir = opendir(FILTERDIR);
+	if (!filterdir) {
+		return;
+	}
+
+	while ((readdir_r(filterdir,&entry,&next) == 0) && next) {
+		if (!strncmp(entry.d_name,".",sizeof(entry.d_name))) {
+			continue;
+		}
+		if (!strncmp(entry.d_name,"..",sizeof(entry.d_name))) {
+			continue;
+		}
+		/*
+		 * d_type isn't guaranteed to be present/valid on all systems,
+		 * so do an explicit stat instead.
+		 */
+		if (gf_asprintf(&filterpath,"%s/%.*s",FILTERDIR,
+				sizeof(entry.d_name), entry.d_name) == (-1)) {
+			continue;
+		}
+		/* Deliberately use stat instead of lstat to allow symlinks. */
+		if (stat(filterpath,&statbuf) == (-1)) {
+			goto free_fp;
+		}
+		if (!S_ISREG(statbuf.st_mode)) {
+			goto free_fp;
+		}
+		/*
+		 * We could check the mode in statbuf directly, or just skip
+		 * this entirely and check for EPERM after exec fails, but this
+		 * is cleaner.
+		 */
+		if (access(filterpath,X_OK) != 0) {
+			goto free_fp;
+		}
+		if (runcmd(filterpath,orig_volfile,NULL)) {
+			gf_log("",GF_LOG_ERROR,"failed to run filter %.*s",
+			       (int)sizeof(entry.d_name), entry.d_name);
+		}
+free_fp:
+		GF_FREE(filterpath);
+	}
+}
+
+static int
+volgen_write_volfile (volgen_graph_t *graph, char *filename)
+{
+        char        *ftmp = NULL;
+        FILE        *f = NULL;
+        int          fd   = 0;
+        xlator_t    *this = NULL;
+
+        this = THIS;
+
+        if (gf_asprintf (&ftmp, "%s.tmp", filename) == -1) {
+                ftmp = NULL;
+
+                goto error;
+        }
+
+        fd = creat (ftmp, S_IRUSR | S_IWUSR);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        strerror (errno));
+                goto error;
+        }
+
+        close (fd);
+
+        f = fopen (ftmp, "w");
+        if (!f)
+                goto error;
+
+        if (glusterfs_graph_print_file (f, &graph->graph) == -1)
+                goto error;
+
+        if (fclose (f) != 0) {
+                gf_log (THIS->name, GF_LOG_ERROR, "fclose on the file %s "
+                        "failed (%s)", ftmp, strerror (errno));
+                /*
+                 * Even though fclose has failed here, we have to set f to NULL.
+                 * Otherwise when the code path goes to error, there again we
+                 * try to close it which might cause undefined behavior such as
+                 * process crash.
+                 */
+                f = NULL;
+                goto error;
+        }
+
+        f = NULL;
+
+        if (rename (ftmp, filename) == -1)
+                goto error;
+
+        GF_FREE (ftmp);
+
+	volgen_apply_filters(filename);
+
+        return 0;
+
+ error:
+
+        GF_FREE (ftmp);
+        if (f)
+                fclose (f);
+
+        gf_log (this->name, GF_LOG_ERROR,
+                "failed to create volfile %s", filename);
+
+        return -1;
+}
+
+static void
+volgen_graph_free (volgen_graph_t *graph)
+{
+        xlator_t *trav = NULL;
+        xlator_t *trav_old = NULL;
+
+        for (trav = first_of (graph) ;; trav = trav->next) {
+                if (trav_old)
+                        xlator_destroy (trav_old);
+
+                trav_old = trav;
+
+                if (!trav)
+                        break;
+        }
+}
+
+static int
+build_graph_generic (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                     dict_t *mod_dict, void *param,
+                     int (*builder) (volgen_graph_t *graph,
+                                     glusterd_volinfo_t *volinfo,
+                                     dict_t *set_dict, void *param))
+{
+        dict_t *set_dict = NULL;
+        int ret = 0;
+
+        if (mod_dict) {
+                set_dict = dict_copy (volinfo->dict, NULL);
+                if (!set_dict)
+                        return -1;
+                 dict_copy (mod_dict, set_dict);
+                 /* XXX dict_copy swallows errors */
+        } else {
+                set_dict = volinfo->dict;
+        }
+
+        ret = builder (graph, volinfo, set_dict, param);
+        if (!ret)
+                ret = volgen_graph_set_options (graph, set_dict);
+
+        if (mod_dict)
+                dict_destroy (set_dict);
+
+        return ret;
+}
+
+static gf_transport_type
+transport_str_to_type (char *tt)
+{
+        gf_transport_type type = GF_TRANSPORT_TCP;
+
+        if (!strcmp ("tcp", tt))
+                type = GF_TRANSPORT_TCP;
+        else if (!strcmp ("rdma", tt))
+                type = GF_TRANSPORT_RDMA;
+        else if (!strcmp ("tcp,rdma", tt))
+                type = GF_TRANSPORT_BOTH_TCP_RDMA;
+        return type;
+}
+
+static void
+transport_type_to_str (gf_transport_type type, char *tt)
+{
+        switch (type) {
+        case GF_TRANSPORT_RDMA:
+                strcpy (tt, "rdma");
+                break;
+        case GF_TRANSPORT_TCP:
+                strcpy (tt, "tcp");
+                break;
+        case GF_TRANSPORT_BOTH_TCP_RDMA:
+                strcpy (tt, "tcp,rdma");
+                break;
+        }
+}
+
+static void
+get_vol_transport_type (glusterd_volinfo_t *volinfo, char *tt)
+{
+        transport_type_to_str (volinfo->transport_type, tt);
+}
+
+static void
+get_vol_nfs_transport_type (glusterd_volinfo_t *volinfo, char *tt)
+{
+        if (volinfo->nfs_transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) {
+                gf_log ("", GF_LOG_ERROR, "%s:nfs transport cannot be both"
+                        " tcp and rdma", volinfo->volname);
+                GF_ASSERT (0);
+        }
+        transport_type_to_str (volinfo->nfs_transport_type, tt);
+}
+
+/*  gets the volinfo, dict, a character array for filling in
+ *  the transport type and a boolean option which says whether
+ *  the transport type is required for nfs or not. If its not
+ *  for nfs, then it is considered as the client transport
+ *  and client transport type is filled in the character array
+ */
+static void
+get_transport_type (glusterd_volinfo_t *volinfo, dict_t *set_dict,
+                    char *transt, gf_boolean_t is_nfs)
+{
+        int    ret   = -1;
+        char  *tt    = NULL;
+        char  *key   = NULL;
+        typedef void (*transport_type) (glusterd_volinfo_t *volinfo, char *tt);
+        transport_type get_transport;
+
+        if (is_nfs == _gf_false) {
+                key = "client-transport-type";
+                get_transport = get_vol_transport_type;
+        } else {
+                key = "nfs.transport-type";
+                get_transport = get_vol_nfs_transport_type;
+        }
+
+        ret = dict_get_str (set_dict, key, &tt);
+        if (ret)
+                get_transport (volinfo, transt);
+        if (!ret)
+                strcpy (transt, tt);
+}
+
+static int
+server_auth_option_handler (volgen_graph_t *graph,
+                            struct volopt_map_entry *vme, void *param)
+{
+        xlator_t *xl = NULL;
+        xlator_list_t *trav = NULL;
+        char *aa = NULL;
+        int   ret   = 0;
+        char *key = NULL;
+
+        if (strcmp (vme->option, "!server-auth") != 0)
+                return 0;
+
+        xl = first_of (graph);
+
+        /* from 'auth.allow' -> 'allow', and 'auth.reject' -> 'reject' */
+        key = strchr (vme->key, '.') + 1;
+
+        for (trav = xl->children; trav; trav = trav->next) {
+                ret = gf_asprintf (&aa, "auth.addr.%s.%s", trav->xlator->name,
+                                   key);
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+                if (ret)
+                        return -1;
+        }
+
+        return 0;
+}
+
+static int
+loglevel_option_handler (volgen_graph_t *graph,
+                         struct volopt_map_entry *vme, void *param)
+{
+        char *role = param;
+        struct volopt_map_entry vme2 = {0,};
+
+        if ( (strcmp (vme->option, "!client-log-level") != 0 &&
+               strcmp (vme->option, "!brick-log-level") != 0)
+            || !strstr (vme->key, role))
+                return 0;
+
+        memcpy (&vme2, vme, sizeof (vme2));
+        vme2.option = "log-level";
+
+        return basic_option_handler (graph, &vme2, NULL);
+}
+
+static int
+server_check_marker_off (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                         glusterd_volinfo_t *volinfo)
+{
+        gf_boolean_t           bool = _gf_false;
+        int                    ret = 0;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (vme);
+
+        if (strcmp (vme->option, "!xtime") != 0)
+                return 0;
+
+        ret = gf_string2boolean (vme->value, &bool);
+        if (ret || bool)
+                goto out;
+
+        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_MARKER_XTIME);
+        if (ret < 0) {
+                gf_log ("", GF_LOG_WARNING, "failed to get the marker status");
+                ret = -1;
+                goto out;
+        }
+
+        if (ret) {
+                bool = _gf_false;
+                ret = glusterd_check_gsync_running (volinfo, &bool);
+
+                if (bool) {
+                        gf_log ("", GF_LOG_WARNING, GEOREP" sessions active"
+                                "for the volume %s, cannot disable marker "
+                                ,volinfo->volname);
+                        set_graph_errstr (graph,
+                                          VKEY_MARKER_XTIME" cannot be disabled "
+                                          "while "GEOREP" sessions exist");
+                        ret = -1;
+                        goto out;
+                }
+
+                if (ret) {
+                        gf_log ("", GF_LOG_WARNING, "Unable to get the status"
+                                 " of active gsync session");
+                        goto out;
+                }
+        }
+
+        ret = 0;
+ out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int
+sys_loglevel_option_handler (volgen_graph_t *graph,
+                             struct volopt_map_entry *vme,
+                             void *param)
+{
+        char  *role = NULL;
+        struct volopt_map_entry vme2 = {0,};
+
+        role = (char *) param;
+
+        if (strcmp (vme->option, "!sys-log-level") != 0 ||
+            !strstr (vme->key, role))
+                return 0;
+
+        memcpy (&vme2, vme, sizeof (vme2));
+        vme2.option = "sys-log-level";
+
+        return basic_option_handler (graph, &vme2, NULL);
+}
+
+static int
+volgen_graph_set_xl_options (volgen_graph_t *graph, dict_t *dict)
+{
+        int32_t   ret               = -1;
+        char     *xlator            = NULL;
+        char     xlator_match[1024] = {0,}; /* for posix* -> *posix* */
+        char     *loglevel          = NULL;
+        xlator_t *trav              = NULL;
+
+        ret = dict_get_str (dict, "xlator", &xlator);
+        if (ret)
+                goto out;
+
+        ret = dict_get_str (dict, "loglevel", &loglevel);
+        if (ret)
+                goto out;
+
+        snprintf (xlator_match, 1024, "*%s", xlator);
+
+        for (trav = first_of (graph); trav; trav = trav->next) {
+                if (fnmatch(xlator_match, trav->type, FNM_NOESCAPE) == 0) {
+                        gf_log ("glusterd", GF_LOG_DEBUG, "Setting log level for xlator: %s",
+                                trav->type);
+                        ret = xlator_set_option (trav, "log-level", loglevel);
+                        if (ret)
+                                break;
+                }
+        }
+
+ out:
+        return ret;
+}
+
+static int
+server_spec_option_handler (volgen_graph_t *graph,
+                            struct volopt_map_entry *vme, void *param)
+{
+        int                     ret = 0;
+        glusterd_volinfo_t      *volinfo = NULL;
+
+        volinfo = param;
+
+        ret = server_auth_option_handler (graph, vme, NULL);
+        if (!ret)
+                ret = server_check_marker_off (graph, vme, volinfo);
+
+        if (!ret)
+                ret = loglevel_option_handler (graph, vme, "brick");
+
+        if (!ret)
+                ret = sys_loglevel_option_handler (graph, vme, "brick");
+
+        return ret;
+}
+
+static int
+server_spec_extended_option_handler (volgen_graph_t *graph,
+                                     struct volopt_map_entry *vme, void *param)
+{
+        int     ret           = 0;
+        dict_t *dict          = NULL;
+
+        GF_ASSERT (param);
+        dict = (dict_t *)param;
+
+        ret = server_auth_option_handler (graph, vme, NULL);
+        if (!ret)
+                ret = volgen_graph_set_xl_options (graph, dict);
+
+        return ret;
+}
+
+static void get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo);
+
+static int
+server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                      dict_t *set_dict, void *param)
+{
+        char     *volname                 = NULL;
+        char     *path                    = NULL;
+        int       pump                    = 0;
+        xlator_t *xl                      = NULL;
+        xlator_t *txl                     = NULL;
+        xlator_t *rbxl                    = NULL;
+        char      transt[16]              = {0,};
+        char     *ptranst                 = NULL;
+        char      volume_id[64]           = {0,};
+        char      tstamp_file[PATH_MAX]   = {0,};
+        int       ret                     = 0;
+        char     *xlator                  = NULL;
+        char     *loglevel                = NULL;
+        char     *username                = NULL;
+        char     *password                = NULL;
+        char     index_basepath[PATH_MAX] = {0};
+        char     key[1024]                = {0};
+        glusterd_brickinfo_t *brickinfo   = NULL;
+        char changelog_basepath[PATH_MAX] = {0,};
+
+        brickinfo = param;
+        path      = brickinfo->path;
+        volname = volinfo->volname;
+        get_vol_transport_type (volinfo, transt);
+
+        ret = dict_get_str (set_dict, "xlator", &xlator);
+
+        /* got a cli log level request */
+        if (!ret) {
+                ret = dict_get_str (set_dict, "loglevel", &loglevel);
+                if (ret) {
+                        gf_log ("glusterd", GF_LOG_ERROR, "could not get both"
+                                " translator name and loglevel for log level request");
+                        goto out;
+                }
+        }
+
+        xl = volgen_graph_add (graph, "storage/posix", volname);
+        if (!xl)
+                return -1;
+
+        ret = xlator_set_option (xl, "directory", path);
+        if (ret)
+                return -1;
+
+        ret = xlator_set_option (xl, "volume-id",
+                                 uuid_utoa (volinfo->volume_id));
+        if (ret)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "posix");
+        if (ret)
+                return -1;
+#ifdef HAVE_BD_XLATOR
+        if (*brickinfo->vg != '\0') {
+                /* Now add BD v2 xlator if volume is BD type */
+                xl = volgen_graph_add (graph, "storage/bd", volname);
+                if (!xl)
+                        return -1;
+
+                ret = xlator_set_option (xl, "device", "vg");
+                if (ret)
+                        return -1;
+                ret = xlator_set_option (xl, "export", brickinfo->vg);
+                if (ret)
+                        return -1;
+
+                ret = check_and_add_debug_xl (graph, set_dict, volname, "bd");
+                if (ret)
+                        return -1;
+
+        }
+#endif
+
+        xl = volgen_graph_add (graph, "features/changelog", volname);
+        if (!xl)
+                return -1;
+
+        ret = xlator_set_option (xl, "changelog-brick", path);
+        if (ret)
+                return -1;
+
+        snprintf (changelog_basepath, sizeof (changelog_basepath),
+                  "%s/%s", path, ".glusterfs/changelogs");
+        ret = xlator_set_option (xl, "changelog-dir", changelog_basepath);
+        if (ret)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "changelog");
+        if (ret)
+                return -1;
+
+        xl = volgen_graph_add (graph, "features/access-control", volname);
+        if (!xl)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "acl");
+        if (ret)
+                return -1;
+
+        xl = volgen_graph_add (graph, "features/locks", volname);
+        if (!xl)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "locks");
+        if (ret)
+                return -1;
+
+        xl = volgen_graph_add (graph, "performance/io-threads", volname);
+        if (!xl)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "io-threads");
+        if (ret)
+                return -1;
+
+        ret = dict_get_int32 (volinfo->dict, "enable-pump", &pump);
+        if (ret == -ENOENT)
+                ret = pump = 0;
+        if (ret)
+                return -1;
+
+        username = glusterd_auth_get_username (volinfo);
+        password = glusterd_auth_get_password (volinfo);
+
+        if (pump) {
+                txl = first_of (graph);
+
+                rbxl = volgen_graph_add_nolink (graph, "protocol/client",
+                                                "%s-replace-brick", volname);
+                if (!rbxl)
+                        return -1;
+
+		ptranst = glusterd_get_trans_type_rb (volinfo->transport_type);
+		if (NULL == ptranst)
+			return -1;
+
+                if (username) {
+                        ret = xlator_set_option (rbxl, "username", username);
+                        if (ret)
+                                return -1;
+                }
+
+                if (password) {
+                        ret = xlator_set_option (rbxl, "password", password);
+                        if (ret)
+                                return -1;
+                }
+
+                ret = xlator_set_option (rbxl, "transport-type", ptranst);
+                GF_FREE (ptranst);
+                if (ret)
+                        return -1;
+
+                xl = volgen_graph_add_nolink (graph, "cluster/pump", "%s-pump",
+                                              volname);
+                if (!xl)
+                        return -1;
+                ret = volgen_xlator_link (xl, txl);
+                if (ret)
+                        return -1;
+                ret = volgen_xlator_link (xl, rbxl);
+                if (ret)
+                        return -1;
+        }
+
+        xl = volgen_graph_add (graph, "features/index", volname);
+        if (!xl)
+                return -1;
+
+        snprintf (index_basepath, sizeof (index_basepath), "%s/%s",
+                  path, ".glusterfs/indices");
+        ret = xlator_set_option (xl, "index-base", index_basepath);
+        if (ret)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "index");
+        if (ret)
+                return -1;
+
+        xl = volgen_graph_add (graph, "features/marker", volname);
+        if (!xl)
+                return -1;
+
+        uuid_unparse (volinfo->volume_id, volume_id);
+        ret = xlator_set_option (xl, "volume-uuid", volume_id);
+        if (ret)
+                return -1;
+        get_vol_tstamp_file (tstamp_file, volinfo);
+        ret = xlator_set_option (xl, "timestamp-file", tstamp_file);
+        if (ret)
+                return -1;
+
+        ret = check_and_add_debug_xl (graph, set_dict, volname, "marker");
+        if (ret)
+                return -1;
+
+        if (dict_get_str_boolean (set_dict, "features.read-only", 0) &&
+            dict_get_str_boolean (set_dict, "features.worm",0)) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "read-only and worm cannot be set together");
+                ret = -1;
+                goto out;
+        }
+
+        /* Check for read-only volume option, and add it to the graph */
+        if (dict_get_str_boolean (set_dict, "features.read-only", 0)
+                || volinfo -> is_snap_volume) {
+                xl = volgen_graph_add (graph, "features/read-only", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Check for worm volume option, and add it to the graph */
+        if (dict_get_str_boolean (set_dict, "features.worm", 0)) {
+                xl = volgen_graph_add (graph, "features/worm", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Check for compress volume option, and add it to the graph on server side */
+        if (dict_get_str_boolean (set_dict, "features.compress", 0)) {
+                xl = volgen_graph_add (graph, "features/cdc", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (set_dict, "compress.mode", "server");
+                if (ret)
+                        goto out;
+        }
+
+        xl = volgen_graph_add_as (graph, "debug/io-stats", path);
+        if (!xl)
+                return -1;
+
+        xl = volgen_graph_add (graph, "protocol/server", volname);
+        if (!xl)
+                return -1;
+        ret = xlator_set_option (xl, "transport-type", transt);
+        if (ret)
+                return -1;
+
+        /*In the case of running multiple glusterds on a single machine,
+         * we should ensure that bricks don't listen on all IPs on that
+         * machine and break the IP based separation being brought about.*/
+        if (dict_get (THIS->options, "transport.socket.bind-address")) {
+                ret = xlator_set_option (xl, "transport.socket.bind-address",
+                                         brickinfo->hostname);
+                if (ret)
+                        return -1;
+        }
+
+        if (username) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "auth.login.%s.allow", path);
+
+                ret = xlator_set_option (xl, key, username);
+                if (ret)
+                        return -1;
+        }
+
+        if (password) {
+                memset (key, 0, sizeof (key));
+                snprintf (key, sizeof (key), "auth.login.%s.password",
+                          username);
+
+                ret = xlator_set_option (xl, key, password);
+                if (ret)
+                        return -1;
+        }
+
+        ret = volgen_graph_set_options_generic (graph, set_dict,
+                                                (xlator && loglevel) ? (void *)set_dict : volinfo,
+                                                (xlator && loglevel) ?  &server_spec_extended_option_handler :
+                                                 &server_spec_option_handler);
+
+ out:
+        return ret;
+}
+
+
+/* builds a graph for server role , with option overrides in mod_dict */
+static int
+build_server_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                    dict_t *mod_dict, glusterd_brickinfo_t *brickinfo)
+{
+        return build_graph_generic (graph, volinfo, mod_dict, brickinfo,
+                                    &server_graph_builder);
+}
+
+static int
+perfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                       void *param)
+{
+        gf_boolean_t enabled = _gf_false;
+        glusterd_volinfo_t *volinfo = NULL;
+
+        GF_ASSERT (param);
+        volinfo = param;
+
+        if (strcmp (vme->option, "!perf") != 0)
+                return 0;
+
+        if (gf_string2boolean (vme->value, &enabled) == -1)
+                return -1;
+        if (!enabled)
+                return 0;
+
+        /* Check op-version before adding the 'open-behind' xlator in the graph
+         */
+        if (!strcmp (vme->key, "performance.open-behind") &&
+            (vme->op_version > volinfo->client_op_version))
+                return 0;
+
+        if (volgen_graph_add (graph, vme->voltype, volinfo->volname))
+                return 0;
+        else
+                return -1;
+}
+
+static int
+nfsperfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                       void *param)
+{
+        char *volname = NULL;
+        gf_boolean_t enabled = _gf_false;
+
+        volname = param;
+
+        if (strcmp (vme->option, "!nfsperf") != 0)
+                return 0;
+
+        if (gf_string2boolean (vme->value, &enabled) == -1)
+                return -1;
+        if (!enabled)
+                return 0;
+
+        if (volgen_graph_add (graph, vme->voltype, volname))
+                return 0;
+        else
+                return -1;
+}
+
+#if (HAVE_LIB_XML)
+static int
+end_sethelp_xml_doc (xmlTextWriterPtr writer)
+{
+        int             ret = -1;
+
+        ret = xmlTextWriterEndElement(writer);
+        if (ret < 0) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not end an "
+                        "xmlElemetnt");
+                ret = -1;
+                goto out;
+        }
+        ret = xmlTextWriterEndDocument (writer);
+        if (ret < 0) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not end an "
+                        "xmlDocument");
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+ out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int
+init_sethelp_xml_doc (xmlTextWriterPtr *writer, xmlBufferPtr  *buf)
+{
+        int ret;
+
+        *buf = xmlBufferCreateSize (8192);
+        if (buf == NULL) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Error creating the xml "
+                          "buffer");
+                ret = -1;
+                goto out;
+        }
+
+        xmlBufferSetAllocationScheme (*buf,XML_BUFFER_ALLOC_DOUBLEIT);
+
+        *writer = xmlNewTextWriterMemory(*buf, 0);
+        if (writer == NULL) {
+                gf_log ("glusterd", GF_LOG_ERROR, " Error creating the xml "
+                         "writer");
+                ret = -1;
+                goto out;
+        }
+
+        ret = xmlTextWriterStartDocument(*writer, "1.0", "UTF-8", "yes");
+        if (ret < 0) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Error While starting the "
+                         "xmlDoc");
+                goto out;
+        }
+
+        ret = xmlTextWriterStartElement(*writer, (xmlChar *)"options");
+        if (ret < 0) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
+                        "xmlElemetnt");
+                ret = -1;
+                goto out;
+        }
+
+
+        ret = 0;
+
+ out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int
+xml_add_volset_element (xmlTextWriterPtr writer, const char *name,
+                                 const char *def_val, const char *dscrpt)
+{
+
+        int                     ret = -1;
+
+        GF_ASSERT (name);
+
+        ret = xmlTextWriterStartElement(writer, (xmlChar *) "option");
+        if (ret < 0) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
+                        "xmlElemetnt");
+                ret = -1;
+                goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar*)"defaultValue",
+                                              "%s", def_val);
+        if (ret < 0) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
+                        "xmlElemetnt");
+                ret = -1;
+                goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"description",
+                                              "%s",  dscrpt );
+        if (ret < 0) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
+                        "xmlElemetnt");
+                ret = -1;
+                goto out;
+        }
+
+        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *) "name", "%s",
+                                               name);
+        if (ret < 0) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
+                        "xmlElemetnt");
+                ret = -1;
+                goto out;
+        }
+
+        ret = xmlTextWriterEndElement(writer);
+        if (ret < 0) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Could not end an "
+                        "xmlElemetnt");
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+ out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+#endif
+
+static int
+_get_xlator_opt_key_from_vme ( struct volopt_map_entry *vme, char **key)
+{
+        int ret = 0;
+
+        GF_ASSERT (vme);
+        GF_ASSERT (key);
+
+
+        if (!strcmp (vme->key, AUTH_ALLOW_MAP_KEY))
+                *key = gf_strdup (AUTH_ALLOW_OPT_KEY);
+        else if (!strcmp (vme->key, AUTH_REJECT_MAP_KEY))
+                *key = gf_strdup (AUTH_REJECT_OPT_KEY);
+        else if (!strcmp (vme->key, NFS_DISABLE_MAP_KEY))
+                *key = gf_strdup (NFS_DISABLE_OPT_KEY);
+        else {
+                if (vme->option) {
+                        if  (vme->option[0] == '!') {
+                                *key = vme->option + 1;
+                                if (!*key[0])
+                                        ret = -1;
+                        } else {
+                                *key = vme->option;
+                        }
+                } else {
+                        *key = strchr (vme->key, '.');
+                        if (*key) {
+                                (*key) ++;
+                                if (!*key[0])
+                                        ret = -1;
+                        } else {
+                                ret = -1;
+                        }
+                }
+        }
+        if (ret)
+                gf_log ("glusterd", GF_LOG_ERROR, "Wrong entry found in  "
+                        "glusterd_volopt_map entry %s", vme->key);
+        else
+                gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static void
+_free_xlator_opt_key (char *key)
+{
+        GF_ASSERT (key);
+
+        if (!strcmp (key, AUTH_ALLOW_OPT_KEY) ||
+            !strcmp (key, AUTH_REJECT_OPT_KEY) ||
+            !strcmp (key, NFS_DISABLE_OPT_KEY))
+                GF_FREE (key);
+
+        return;
+}
+
+int
+glusterd_get_volopt_content (dict_t * ctx, gf_boolean_t xml_out)
+{
+        void                    *dl_handle = NULL;
+        volume_opt_list_t        vol_opt_handle = {{0},};
+        char                    *key = NULL;
+        struct volopt_map_entry *vme = NULL;
+        int                      ret = -1;
+        char                    *def_val = NULL;
+        char                    *descr = NULL;
+        char                     output_string[25600] = {0, };
+        char                    *output = NULL;
+        char                     tmp_str[2048] = {0, };
+#if (HAVE_LIB_XML)
+        xmlTextWriterPtr         writer = NULL;
+        xmlBufferPtr             buf = NULL;
+
+        if (xml_out) {
+                ret = init_sethelp_xml_doc (&writer, &buf);
+                if (ret) /*logging done in init_xml_lib*/
+                        goto out;
+        }
+#endif
+
+        INIT_LIST_HEAD (&vol_opt_handle.list);
+
+        for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
+
+                if ((vme->type == NO_DOC) || (vme->type == GLOBAL_NO_DOC))
+                        continue;
+
+                if (vme->description) {
+                        descr = vme->description;
+                        def_val = vme->value;
+                } else {
+                        if (_get_xlator_opt_key_from_vme (vme, &key)) {
+                                gf_log ("glusterd", GF_LOG_DEBUG, "Failed to "
+                                        "get %s key from volume option entry",
+                                        vme->key);
+                                goto out; /*Some error while geting key*/
+                        }
+
+                        ret = xlator_volopt_dynload (vme->voltype,
+                                                     &dl_handle,
+                                                     &vol_opt_handle);
+
+                        if (ret) {
+                                gf_log ("glusterd", GF_LOG_DEBUG,
+                                        "xlator_volopt_dynload error(%d)", ret);
+                                ret = 0;
+                                goto cont;
+                        }
+
+                        ret = xlator_option_info_list (&vol_opt_handle, key,
+                                                       &def_val, &descr);
+                        if (ret) { /*Swallow Error i.e if option not found*/
+                                gf_log ("glusterd", GF_LOG_DEBUG,
+                                        "Failed to get option for %s key", key);
+                                ret = 0;
+                                goto cont;
+                        }
+                }
+
+                if (xml_out) {
+#if (HAVE_LIB_XML)
+                        if (xml_add_volset_element (writer,vme->key,
+                                                    def_val, descr)) {
+                                ret = -1;
+                                goto cont;
+                        }
+#else
+                        gf_log ("glusterd", GF_LOG_ERROR, "Libxml not present");
+#endif
+                } else {
+                        snprintf (tmp_str, sizeof (tmp_str), "Option: %s\nDefault "
+                                        "Value: %s\nDescription: %s\n\n",
+                                        vme->key, def_val, descr);
+                        strcat (output_string, tmp_str);
+                }
+cont:
+                if (dl_handle) {
+                        dlclose (dl_handle);
+                        dl_handle = NULL;
+                        vol_opt_handle.given_opt = NULL;
+                }
+                if (key) {
+                        _free_xlator_opt_key (key);
+                        key = NULL;
+                }
+                if (ret)
+                        goto out;
+        }
+
+#if (HAVE_LIB_XML)
+        if ((xml_out) &&
+            (ret = end_sethelp_xml_doc (writer)))
+                goto out;
+#else
+        if (xml_out)
+                gf_log ("glusterd", GF_LOG_ERROR, "Libxml not present");
+#endif
+
+        if (!xml_out)
+                output = gf_strdup (output_string);
+        else
+#if (HAVE_LIB_XML)
+                output = gf_strdup ((char *)buf->content);
+#else
+                gf_log ("glusterd", GF_LOG_ERROR, "Libxml not present");
+#endif
+
+        if (NULL == output) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_dynstr (ctx, "help-str", output);
+out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int
+volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                            dict_t *set_dict, void *param)
+{
+        int                      i                  = 0;
+        int                      ret                = -1;
+        uint32_t                 client_type        = GF_CLIENT_OTHER;
+        char                     transt[16]         = {0,};
+        char                    *volname            = NULL;
+        char                    *str                = NULL;
+        glusterd_brickinfo_t    *brick              = NULL;
+        xlator_t                *xl                 = NULL;
+        char                    *ssl_str            = NULL;
+        gf_boolean_t             ssl_bool;
+
+        volname = volinfo->volname;
+
+        if (volinfo->brick_count == 0) {
+                gf_log ("", GF_LOG_ERROR,
+                        "volume inconsistency: brick count is 0");
+                goto out;
+        }
+
+        if ((volinfo->dist_leaf_count < volinfo->brick_count) &&
+            ((volinfo->brick_count % volinfo->dist_leaf_count) != 0)) {
+                gf_log ("", GF_LOG_ERROR,
+                        "volume inconsistency: "
+                        "total number of bricks (%d) is not divisible with "
+                        "number of bricks per cluster (%d) in a multi-cluster "
+                        "setup",
+                        volinfo->brick_count, volinfo->dist_leaf_count);
+                goto out;
+        }
+
+        get_transport_type (volinfo, set_dict, transt, _gf_false);
+
+        if (!strcmp (transt, "tcp,rdma"))
+                strcpy (transt, "tcp");
+
+        i = 0;
+        ret = -1;
+        list_for_each_entry (brick, &volinfo->bricks, brick_list) {
+                ret = -1;
+                xl = volgen_graph_add_nolink (graph, "protocol/client",
+                                              "%s-client-%d", volname, i);
+                if (!xl)
+                        goto out;
+                ret = xlator_set_option (xl, "remote-host", brick->hostname);
+                if (ret)
+                        goto out;
+                ret = xlator_set_option (xl, "remote-subvolume", brick->path);
+                if (ret)
+                        goto out;
+                ret = xlator_set_option (xl, "transport-type", transt);
+                if (ret)
+                        goto out;
+
+                ret = dict_get_uint32 (set_dict, "trusted-client",
+                                       &client_type);
+
+                if (!ret && client_type == GF_CLIENT_TRUSTED) {
+                        str = NULL;
+                        str = glusterd_auth_get_username (volinfo);
+                        if (str) {
+                                ret = xlator_set_option (xl, "username",
+                                                         str);
+                                if (ret)
+                                        goto out;
+                        }
+
+                        str = glusterd_auth_get_password (volinfo);
+                        if (str) {
+                                ret = xlator_set_option (xl, "password",
+                                                         str);
+                                if (ret)
+                                        goto out;
+                        }
+                }
+
+                if (dict_get_str(set_dict,"client.ssl",&ssl_str) == 0) {
+                        if (gf_string2boolean(ssl_str,&ssl_bool) == 0) {
+                                if (ssl_bool) {
+                                        ret = xlator_set_option(xl,
+                                                "transport.socket.ssl-enabled",
+                                                "true");
+                                        if (ret) {
+                                                goto out;
+                                        }
+                                }
+                        }
+                }
+
+                i++;
+        }
+
+        if (i != volinfo->brick_count) {
+                gf_log ("", GF_LOG_ERROR,
+                        "volume inconsistency: actual number of bricks (%d) "
+                        "differs from brick count (%d)", i,
+                        volinfo->brick_count);
+
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+volgen_graph_build_clusters (volgen_graph_t *graph,
+                             glusterd_volinfo_t *volinfo, char *xl_type,
+                             char *xl_namefmt, size_t child_count,
+                             size_t sub_count)
+{
+        int             i = 0;
+        int             j = 0;
+        xlator_t        *txl = NULL;
+        xlator_t        *xl  = NULL;
+        xlator_t        *trav = NULL;
+        char            *volname = NULL;
+        int             ret     = -1;
+
+        if (child_count == 0)
+                goto out;
+        volname = volinfo->volname;
+        txl = first_of (graph);
+        for (trav = txl; --child_count; trav = trav->next);
+        for (;; trav = trav->prev) {
+                if ((i % sub_count) == 0) {
+                        xl = volgen_graph_add_nolink (graph, xl_type,
+                                                      xl_namefmt, volname, j);
+                        if (!xl) {
+                                ret = -1;
+                                goto out;
+                        }
+                        j++;
+                }
+
+                ret = volgen_xlator_link (xl, trav);
+                if (ret)
+                        goto out;
+
+                if (trav == txl)
+                        break;
+
+                i++;
+        }
+
+        ret = j;
+out:
+        return ret;
+}
+
+gf_boolean_t
+_xl_is_client_decommissioned (xlator_t *xl, glusterd_volinfo_t *volinfo)
+{
+        int             ret = 0;
+        gf_boolean_t    decommissioned = _gf_false;
+        char            *hostname = NULL;
+        char            *path = NULL;
+
+        GF_ASSERT (!strcmp (xl->type, "protocol/client"));
+        ret = xlator_get_option (xl, "remote-host", &hostname);
+        if (ret) {
+                GF_ASSERT (0);
+                gf_log ("glusterd", GF_LOG_ERROR, "Failed to get remote-host "
+                        "from client %s", xl->name);
+                goto out;
+        }
+        ret = xlator_get_option (xl, "remote-subvolume", &path);
+        if (ret) {
+                GF_ASSERT (0);
+                gf_log ("glusterd", GF_LOG_ERROR, "Failed to get remote-host "
+                        "from client %s", xl->name);
+                goto out;
+        }
+
+        decommissioned = glusterd_is_brick_decommissioned (volinfo, hostname,
+                                                           path);
+out:
+        return decommissioned;
+}
+
+gf_boolean_t
+_xl_has_decommissioned_clients (xlator_t *xl, glusterd_volinfo_t *volinfo)
+{
+        xlator_list_t   *xl_child = NULL;
+        gf_boolean_t    decommissioned = _gf_false;
+        xlator_t        *cxl = NULL;
+
+        if (!xl)
+                goto out;
+
+        if (!strcmp (xl->type, "protocol/client")) {
+                decommissioned = _xl_is_client_decommissioned (xl, volinfo);
+                goto out;
+        }
+
+        xl_child = xl->children;
+        while (xl_child) {
+                cxl = xl_child->xlator;
+                /* this can go into 2 depths if the volume type
+                   is stripe-replicate */
+                decommissioned = _xl_has_decommissioned_clients (cxl, volinfo);
+                if (decommissioned)
+                        break;
+
+                xl_child = xl_child->next;
+        }
+out:
+        return decommissioned;
+}
+
+static int
+_graph_get_decommissioned_children (xlator_t *dht, glusterd_volinfo_t *volinfo,
+                                    char **children)
+{
+        int             ret = -1;
+        xlator_list_t   *xl_child = NULL;
+        xlator_t        *cxl = NULL;
+        gf_boolean_t    comma = _gf_false;
+
+        *children = NULL;
+        xl_child = dht->children;
+        while (xl_child) {
+                cxl = xl_child->xlator;
+                if (_xl_has_decommissioned_clients (cxl, volinfo)) {
+                        if (!*children) {
+                                *children = GF_CALLOC (16 * GF_UNIT_KB, 1,
+                                                       gf_common_mt_char);
+                                if (!*children)
+                                        goto out;
+                        }
+
+                        if (comma)
+                                strcat (*children, ",");
+                        strcat (*children, cxl->name);
+                        comma = _gf_true;
+                }
+
+                xl_child = xl_child->next;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+volgen_graph_build_dht_cluster (volgen_graph_t *graph,
+                                glusterd_volinfo_t *volinfo, size_t child_count)
+{
+        int32_t                 clusters                 = 0;
+        int                     ret                      = -1;
+        char                    *decommissioned_children = NULL;
+        xlator_t                *dht                     = NULL;
+        char                    *voltype                 = "cluster/distribute";
+
+        /* NUFA and Switch section */
+        if (dict_get_str_boolean (volinfo->dict, "cluster.nufa", 0) &&
+            dict_get_str_boolean (volinfo->dict, "cluster.switch", 0)) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "nufa and switch cannot be set together");
+                ret = -1;
+                goto out;
+        }
+
+        /* Check for NUFA volume option, and change the voltype */
+        if (dict_get_str_boolean (volinfo->dict, "cluster.nufa", 0))
+                voltype = "cluster/nufa";
+
+        /* Check for switch volume option, and change the voltype */
+        if (dict_get_str_boolean (volinfo->dict, "cluster.switch", 0))
+                voltype = "cluster/switch";
+
+        clusters = volgen_graph_build_clusters (graph,  volinfo,
+                                                voltype,
+                                                "%s-dht",
+                                                child_count,
+                                                child_count);
+        if (clusters < 0)
+                goto out;
+
+        dht = first_of (graph);
+        ret = _graph_get_decommissioned_children (dht, volinfo,
+                                                  &decommissioned_children);
+        if (ret)
+                goto out;
+        if (decommissioned_children) {
+                ret = xlator_set_option (dht, "decommissioned-bricks",
+                                         decommissioned_children);
+                if (ret)
+                        goto out;
+        }
+        ret = 0;
+out:
+        GF_FREE (decommissioned_children);
+        return ret;
+}
+
+static int
+volume_volgen_graph_build_clusters (volgen_graph_t *graph,
+                                    glusterd_volinfo_t *volinfo)
+{
+        char                    *replicate_args[]   = {"cluster/replicate",
+                                                       "%s-replicate-%d"};
+        char                    *stripe_args[]      = {"cluster/stripe",
+                                                       "%s-stripe-%d"};
+        int                     rclusters           = 0;
+        int                     clusters            = 0;
+        int                     dist_count          = 0;
+        int                     ret                 = -1;
+
+        if (!volinfo->dist_leaf_count)
+                goto out;
+
+        if (volinfo->dist_leaf_count == 1)
+                goto build_distribute;
+
+        /* All other cases, it will have one or the other cluster type */
+        switch (volinfo->type) {
+        case GF_CLUSTER_TYPE_REPLICATE:
+                clusters = volgen_graph_build_clusters (graph, volinfo,
+                                                        replicate_args[0],
+                                                        replicate_args[1],
+                                                        volinfo->brick_count,
+                                                        volinfo->replica_count);
+                if (clusters < 0)
+                        goto out;
+                break;
+        case GF_CLUSTER_TYPE_STRIPE:
+                clusters = volgen_graph_build_clusters (graph, volinfo,
+                                                        stripe_args[0],
+                                                        stripe_args[1],
+                                                        volinfo->brick_count,
+                                                        volinfo->stripe_count);
+                if (clusters < 0)
+                        goto out;
+                break;
+        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                /* Replicate after the clients, then stripe */
+                if (volinfo->replica_count == 0)
+                        goto out;
+                clusters = volgen_graph_build_clusters (graph, volinfo,
+                                                        replicate_args[0],
+                                                        replicate_args[1],
+                                                        volinfo->brick_count,
+                                                        volinfo->replica_count);
+                if (clusters < 0)
+                        goto out;
+
+                rclusters = volinfo->brick_count / volinfo->replica_count;
+                GF_ASSERT (rclusters == clusters);
+                clusters = volgen_graph_build_clusters (graph, volinfo,
+                                                        stripe_args[0],
+                                                        stripe_args[1],
+                                                        rclusters,
+                                                        volinfo->stripe_count);
+                if (clusters < 0)
+                        goto out;
+                break;
+        default:
+                gf_log ("", GF_LOG_ERROR, "volume inconsistency: "
+                        "unrecognized clustering type");
+                goto out;
+        }
+
+build_distribute:
+        dist_count = volinfo->brick_count / volinfo->dist_leaf_count;
+        if (!dist_count) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = volgen_graph_build_dht_cluster (graph, volinfo,
+                                              dist_count);
+        if (ret == -1)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int client_graph_set_perf_options(volgen_graph_t *graph,
+					 glusterd_volinfo_t *volinfo,
+					 dict_t *set_dict)
+{
+	data_t *tmp_data = NULL;
+	char *volname = NULL;
+
+	/*
+	 * Logic to make sure NFS doesn't have performance translators by
+	 * default for a volume
+	 */
+	volname = volinfo->volname;
+	tmp_data = dict_get (set_dict, "nfs-volume-file");
+	if (!tmp_data)
+		return volgen_graph_set_options_generic(graph, set_dict,
+							volname,
+							&perfxl_option_handler);
+	else
+		return volgen_graph_set_options_generic(graph, set_dict,
+							volname,
+							&nfsperfxl_option_handler);
+}
+
+static int
+client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                      dict_t *set_dict, void *param)
+{
+        int       ret      = 0;
+        xlator_t *xl       = NULL;
+        char     *volname  = NULL;
+
+        volname = volinfo->volname;
+        ret = volgen_graph_build_clients (graph, volinfo, set_dict, param);
+        if (ret)
+                goto out;
+
+        ret = volume_volgen_graph_build_clusters (graph, volinfo);
+        if (ret == -1)
+                goto out;
+
+        /* Check for compress volume option, and add it to the graph on client side */
+        if (dict_get_str_boolean (set_dict, "features.compress", 0)) {
+                xl = volgen_graph_add (graph, "features/cdc", volname);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = dict_set_str (set_dict, "compress.mode", "client");
+                if (ret)
+                        goto out;
+
+        }
+
+        ret = glusterd_volinfo_get_boolean (volinfo, "features.encryption");
+        if (ret == -1)
+                goto out;
+        if (ret) {
+                xl = volgen_graph_add (graph, "encryption/crypt", volname);
+
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
+        if (ret == -1)
+                goto out;
+        if (ret) {
+                xl = volgen_graph_add (graph, "features/quota", volname);
+
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+
+        ret = glusterd_volinfo_get_boolean (volinfo, "features.file-snapshot");
+        if (ret == -1)
+                goto out;
+        if (ret) {
+                xl = volgen_graph_add (graph, "features/qemu-block", volname);
+
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+	ret = client_graph_set_perf_options(graph, volinfo, set_dict);
+        if (ret)
+                goto out;
+
+        /* add debug translators depending on the options */
+        ret = check_and_add_debug_xl (graph, set_dict, volname,
+                                      "client");
+        if (ret)
+                return -1;
+
+        ret = -1;
+        xl = volgen_graph_add_as (graph, "debug/io-stats", volname);
+        if (!xl)
+                goto out;
+
+        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
+                                                &loglevel_option_handler);
+
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING, "changing client log level"
+                        " failed");
+
+        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
+                                                &sys_loglevel_option_handler);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING, "changing client syslog "
+                        "level failed");
+out:
+        return ret;
+}
+
+
+/* builds a graph for client role , with option overrides in mod_dict */
+static int
+build_client_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                    dict_t *mod_dict)
+{
+        return build_graph_generic (graph, volinfo, mod_dict, NULL,
+                                    &client_graph_builder);
+}
+
+char *gd_shd_options[] = {
+        "!self-heal-daemon",
+        "!heal-timeout",
+        NULL
+};
+
+char*
+gd_get_matching_option (char **options, char *option)
+{
+        while (*options && strcmp (*options, option))
+                options++;
+        return *options;
+}
+
+static int
+shd_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+                    void *param)
+{
+        int                     ret = 0;
+        struct volopt_map_entry new_vme = {0};
+        char                    *shd_option = NULL;
+
+        shd_option = gd_get_matching_option (gd_shd_options, vme->option);
+        if ((vme->option[0] == '!') && !shd_option)
+                goto out;
+        new_vme = *vme;
+        if (shd_option) {
+                new_vme.option = shd_option + 1;//option with out '!'
+        }
+
+        ret = no_filter_option_handler (graph, &new_vme, param);
+out:
+        return ret;
+}
+
+static int
+nfs_option_handler (volgen_graph_t *graph,
+                    struct volopt_map_entry *vme, void *param)
+{
+        xlator_t *xl = NULL;
+        char *aa = NULL;
+        int   ret   = 0;
+        glusterd_volinfo_t *volinfo = NULL;
+
+        volinfo = param;
+
+        xl = first_of (graph);
+
+/*        if (vme->type  == GLOBAL_DOC || vme->type == GLOBAL_NO_DOC) {
+
+                ret = xlator_set_option (xl, vme->key, vme->value);
+        }*/
+        if (!volinfo || (volinfo->volname[0] == '\0'))
+                return 0;
+
+        if (! strcmp (vme->option, "!rpc-auth.addr.*.allow")) {
+                ret = gf_asprintf (&aa, "rpc-auth.addr.%s.allow",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+
+        if (! strcmp (vme->option, "!rpc-auth.addr.*.reject")) {
+                ret = gf_asprintf (&aa, "rpc-auth.addr.%s.reject",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+
+        if (! strcmp (vme->option, "!rpc-auth.auth-unix.*")) {
+                ret = gf_asprintf (&aa, "rpc-auth.auth-unix.%s",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+        if (! strcmp (vme->option, "!rpc-auth.auth-null.*")) {
+                ret = gf_asprintf (&aa, "rpc-auth.auth-null.%s",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+
+        if (! strcmp (vme->option, "!nfs3.*.trusted-sync")) {
+                ret = gf_asprintf (&aa, "nfs3.%s.trusted-sync",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+
+        if (! strcmp (vme->option, "!nfs3.*.trusted-write")) {
+                ret = gf_asprintf (&aa, "nfs3.%s.trusted-write",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+
+        if (! strcmp (vme->option, "!nfs3.*.volume-access")) {
+                ret = gf_asprintf (&aa, "nfs3.%s.volume-access",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+
+        if (! strcmp (vme->option, "!nfs3.*.export-dir")) {
+                ret = gf_asprintf (&aa, "nfs3.%s.export-dir",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = gf_canonicalize_path (vme->value);
+                        if (ret)
+                                return -1;
+
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+
+
+
+        if (! strcmp (vme->option, "!rpc-auth.ports.*.insecure")) {
+                ret = gf_asprintf (&aa, "rpc-auth.ports.%s.insecure",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+
+
+        if (! strcmp (vme->option, "!nfs-disable")) {
+                ret = gf_asprintf (&aa, "nfs.%s.disable",
+                                        volinfo->volname);
+
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+
+                if (ret)
+                        return -1;
+        }
+
+        if ( (strcmp (vme->voltype, "nfs/server") == 0) &&
+             (vme->option && vme->option[0]!='!') ) {
+               ret = xlator_set_option (xl, vme->option, vme->value);
+                if (ret)
+                        return -1;
+        }
+
+
+        /*key = strchr (vme->key, '.') + 1;
+
+        for (trav = xl->children; trav; trav = trav->next) {
+                ret = gf_asprintf (&aa, "auth.addr.%s.%s", trav->xlator->name,
+                                   key);
+                if (ret != -1) {
+                        ret = xlator_set_option (xl, aa, vme->value);
+                        GF_FREE (aa);
+                }
+                if (ret)
+                        return -1;
+        }*/
+
+        return 0;
+}
+
+static int
+volgen_graph_set_iam_shd (volgen_graph_t *graph)
+{
+        xlator_t        *trav;
+        int             ret = 0;
+
+        for (trav = first_of (graph); trav; trav = trav->next) {
+                if (strcmp (trav->type, "cluster/replicate") != 0)
+                        continue;
+
+                ret = xlator_set_option (trav, "iam-self-heal-daemon", "yes");
+                if (ret)
+                        break;
+        }
+        return ret;
+}
+
+static int
+build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
+{
+        volgen_graph_t     cgraph         = {0};
+        glusterd_volinfo_t *voliter       = NULL;
+        xlator_t           *this          = NULL;
+        glusterd_conf_t    *priv          = NULL;
+        dict_t             *set_dict      = NULL;
+        int                ret            = 0;
+        gf_boolean_t       valid_config   = _gf_false;
+        xlator_t           *iostxl        = NULL;
+        int                rclusters      = 0;
+        int                replica_count  = 0;
+        gf_boolean_t       graph_check    = _gf_false;
+
+        this = THIS;
+        priv = this->private;
+
+        set_dict = dict_new ();
+        if (!set_dict) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        graph_check = dict_get_str_boolean (mod_dict, "graph-check", 0);
+        iostxl = volgen_graph_add_as (graph, "debug/io-stats", "glustershd");
+        if (!iostxl) {
+                ret = -1;
+                goto out;
+        }
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (!graph_check &&
+                   (voliter->status != GLUSTERD_STATUS_STARTED))
+                        continue;
+
+                if (!glusterd_is_volume_replicate (voliter))
+                        continue;
+
+                replica_count = voliter->replica_count;
+
+                valid_config = _gf_true;
+
+                ret = dict_set_str (set_dict, "cluster.self-heal-daemon", "on");
+                if (ret)
+                        goto out;
+
+                ret = dict_set_uint32 (set_dict, "trusted-client",
+                                       GF_CLIENT_TRUSTED);
+                if (ret)
+                        goto out;
+
+                dict_copy (voliter->dict, set_dict);
+                if (mod_dict)
+                        dict_copy (mod_dict, set_dict);
+
+                memset (&cgraph, 0, sizeof (cgraph));
+                ret = volgen_graph_build_clients (&cgraph, voliter, set_dict,
+                                                  NULL);
+                if (ret)
+                        goto out;
+
+                rclusters = volgen_graph_build_clusters (&cgraph, voliter,
+                                                        "cluster/replicate",
+                                                        "%s-replicate-%d",
+                                                        voliter->brick_count,
+                                                        replica_count);
+                if (rclusters < 0) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = volgen_graph_set_options_generic (&cgraph, set_dict, voliter,
+                                                        shd_option_handler);
+                if (ret)
+                        goto out;
+
+                ret = volgen_graph_set_iam_shd (&cgraph);
+                if (ret)
+                        goto out;
+
+                ret = volgen_graph_merge_sub (graph, &cgraph, rclusters);
+                if (ret)
+                        goto out;
+
+                ret = volgen_graph_set_options_generic (graph, set_dict,
+                                                        "client",
+                                                 &loglevel_option_handler);
+
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING, "changing loglevel "
+                                "of self-heal daemon failed");
+
+                ret = volgen_graph_set_options_generic (graph, set_dict,
+                                                        "client",
+                                                 &sys_loglevel_option_handler);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING, "changing syslog "
+                                "level of self-heal daemon failed");
+
+                ret = dict_reset (set_dict);
+                if (ret)
+                        goto out;
+        }
+out:
+        if (set_dict)
+                dict_unref (set_dict);
+        if (!valid_config)
+                ret = -EINVAL;
+        return ret;
+}
+
+/* builds a graph for nfs server role, with option overrides in mod_dict */
+static int
+build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
+{
+        volgen_graph_t      cgraph        = {0,};
+        glusterd_volinfo_t *voliter       = NULL;
+        xlator_t           *this          = NULL;
+        glusterd_conf_t    *priv          = NULL;
+        dict_t             *set_dict      = NULL;
+        xlator_t           *nfsxl         = NULL;
+        char               *skey          = NULL;
+        int                 ret           = 0;
+        char               nfs_xprt[16]   = {0,};
+        char               *volname       = NULL;
+        data_t             *data          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        set_dict = dict_new ();
+        if (!set_dict) {
+                gf_log ("", GF_LOG_ERROR, "Out of memory");
+                return -1;
+        }
+
+        nfsxl = volgen_graph_add_as (graph, "nfs/server", "nfs-server");
+        if (!nfsxl) {
+                ret = -1;
+                goto out;
+        }
+        ret = xlator_set_option (nfsxl, "nfs.dynamic-volumes", "on");
+        if (ret)
+                goto out;
+
+        ret = xlator_set_option (nfsxl, "nfs.nlm", "on");
+        if (ret)
+                goto out;
+
+        ret = xlator_set_option (nfsxl, "nfs.drc", "on");
+        if (ret)
+                goto out;
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+                if (voliter->status != GLUSTERD_STATUS_STARTED)
+                        continue;
+
+                if (dict_get_str_boolean (voliter->dict, "nfs.disable", 0))
+                        continue;
+
+                ret = gf_asprintf (&skey, "rpc-auth.addr.%s.allow",
+                                   voliter->volname);
+                if (ret == -1) {
+                        gf_log ("", GF_LOG_ERROR, "Out of memory");
+                        goto out;
+                }
+                ret = xlator_set_option (nfsxl, skey, "*");
+                GF_FREE (skey);
+                if (ret)
+                        goto out;
+
+                ret = gf_asprintf (&skey, "nfs3.%s.volume-id",
+                                   voliter->volname);
+                if (ret == -1) {
+                        gf_log ("", GF_LOG_ERROR, "Out of memory");
+                        goto out;
+                }
+                ret = xlator_set_option (nfsxl, skey, uuid_utoa (voliter->volume_id));
+                GF_FREE (skey);
+                if (ret)
+                        goto out;
+
+                /* If both RDMA and TCP are the transport_type, use RDMA
+                   for NFS client protocols */
+                memset (&cgraph, 0, sizeof (cgraph));
+                if (mod_dict)
+                        get_transport_type (voliter, mod_dict, nfs_xprt, _gf_true);
+                else
+                        get_transport_type (voliter, voliter->dict, nfs_xprt, _gf_true);
+
+                ret = dict_set_str (set_dict, "performance.stat-prefetch", "off");
+                if (ret)
+                        goto out;
+
+                ret = dict_set_str (set_dict, "performance.client-io-threads",
+                                    "off");
+                if (ret)
+                        goto out;
+
+                ret = dict_set_str (set_dict, "client-transport-type",
+                                    nfs_xprt);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_uint32 (set_dict, "trusted-client",
+                                       GF_CLIENT_TRUSTED);
+                if (ret)
+                        goto out;
+
+                ret = dict_set_str (set_dict, "nfs-volume-file", "yes");
+                if (ret)
+                        goto out;
+
+                if (mod_dict && (data = dict_get (mod_dict, "volume-name"))) {
+                        volname = data->data;
+                        if (strcmp (volname, voliter->volname) == 0)
+                                dict_copy (mod_dict, set_dict);
+                }
+
+                ret = build_client_graph (&cgraph, voliter, set_dict);
+                if (ret)
+                        goto out;
+
+                if (mod_dict) {
+                        dict_copy (mod_dict, set_dict);
+                        ret = volgen_graph_set_options_generic (&cgraph, set_dict, voliter,
+                                                                basic_option_handler);
+                } else {
+                        ret = volgen_graph_set_options_generic (&cgraph, voliter->dict, voliter,
+                                                                basic_option_handler);
+                }
+
+                if (ret)
+                        goto out;
+
+                ret = volgen_graph_merge_sub (graph, &cgraph, 1);
+                if (ret)
+                        goto out;
+                ret = dict_reset (set_dict);
+                if (ret)
+                        goto out;
+        }
+
+        list_for_each_entry (voliter, &priv->volumes, vol_list) {
+
+                if (mod_dict) {
+                        ret = volgen_graph_set_options_generic (graph, mod_dict, voliter,
+                                                                nfs_option_handler);
+                } else {
+                        ret = volgen_graph_set_options_generic (graph, voliter->dict, voliter,
+                                                                nfs_option_handler);
+                }
+
+                if (ret)
+                        gf_log ("glusterd", GF_LOG_WARNING, "Could not set "
+                                 "vol-options for the volume %s", voliter->volname);
+        }
+
+ out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        dict_destroy (set_dict);
+
+        return ret;
+}
+
+
+
+
+/****************************
+ *
+ * Volume generation interface
+ *
+ ****************************/
+
+
+static void
+get_brick_filepath (char *filename, glusterd_volinfo_t *volinfo,
+                    glusterd_brickinfo_t *brickinfo)
+{
+        char  path[PATH_MAX]   = {0,};
+        char  brick[PATH_MAX]  = {0,};
+        glusterd_conf_t *priv  = NULL;
+
+        priv = THIS->private;
+
+        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, brick);
+        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+
+        snprintf (filename, PATH_MAX, "%s/%s.%s.%s.vol",
+                  path, volinfo->volname,
+                  brickinfo->hostname,
+                  brick);
+}
+
+gf_boolean_t
+glusterd_is_valid_volfpath (char *volname, char *brick)
+{
+        char                    volfpath[PATH_MAX] = {0,};
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        int32_t                 ret = 0;
+        xlator_t                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create brickinfo"
+                        " for brick %s", brick );
+                ret = 0;
+                goto out;
+        }
+        ret = glusterd_volinfo_new (&volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create volinfo");
+                ret = 0;
+                goto out;
+        }
+        strncpy (volinfo->volname, volname, sizeof (volinfo->volname));
+        get_brick_filepath (volfpath, volinfo, brickinfo);
+
+        ret = (strlen (volfpath) < _POSIX_PATH_MAX);
+
+out:
+        if (brickinfo)
+                glusterd_brickinfo_delete (brickinfo);
+        if (volinfo)
+                glusterd_volinfo_delete (volinfo);
+        return ret;
+}
+
+static int
+glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo,
+                                 glusterd_brickinfo_t *brickinfo)
+{
+        volgen_graph_t graph = {0,};
+        char    filename[PATH_MAX] = {0,};
+        int     ret = -1;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+
+        get_brick_filepath (filename, volinfo, brickinfo);
+
+        ret = build_server_graph (&graph, volinfo, NULL, brickinfo);
+        if (!ret)
+                ret = volgen_write_volfile (&graph, filename);
+
+        volgen_graph_free (&graph);
+
+        return ret;
+}
+
+static void
+get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo)
+{
+        glusterd_conf_t *priv  = NULL;
+
+        priv = THIS->private;
+
+        GLUSTERD_GET_VOLUME_DIR (filename, volinfo, priv);
+        strncat (filename, "/marker.tstamp",
+                 PATH_MAX - strlen(filename) - 1);
+}
+
+int
+generate_brick_volfiles (glusterd_volinfo_t *volinfo)
+{
+        glusterd_brickinfo_t    *brickinfo = NULL;
+        char                     tstamp_file[PATH_MAX] = {0,};
+        int                      ret = -1;
+
+        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_MARKER_XTIME);
+        if (ret == -1)
+                return -1;
+
+        get_vol_tstamp_file (tstamp_file, volinfo);
+
+        if (ret) {
+                ret = open (tstamp_file, O_WRONLY|O_CREAT|O_EXCL, 0600);
+                if (ret == -1 && errno == EEXIST) {
+                        gf_log ("", GF_LOG_DEBUG, "timestamp file exist");
+                        ret = -2;
+                }
+                if (ret == -1) {
+                        gf_log ("", GF_LOG_ERROR, "failed to create %s (%s)",
+                                tstamp_file, strerror (errno));
+                        return -1;
+                }
+                if (ret >= 0)
+                        close (ret);
+        } else {
+                ret = unlink (tstamp_file);
+                if (ret == -1 && errno == ENOENT)
+                        ret = 0;
+                if (ret == -1) {
+                        gf_log ("", GF_LOG_ERROR, "failed to unlink %s (%s)",
+                                tstamp_file, strerror (errno));
+                        return -1;
+                }
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "Found a brick - %s:%s", brickinfo->hostname,
+                        brickinfo->path);
+
+                ret = glusterd_generate_brick_volfile (volinfo, brickinfo);
+                if (ret)
+                        goto out;
+
+        }
+
+        ret = 0;
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+generate_single_transport_client_volfile (glusterd_volinfo_t *volinfo,
+                                          char *filepath, dict_t *dict)
+{
+        volgen_graph_t graph = {0,};
+        int     ret = -1;
+
+        ret = build_client_graph (&graph, volinfo, dict);
+        if (!ret)
+                ret = volgen_write_volfile (&graph, filepath);
+
+        volgen_graph_free (&graph);
+
+        return ret;
+}
+
+static void
+enumerate_transport_reqs (gf_transport_type type, char **types)
+{
+        switch (type) {
+        case GF_TRANSPORT_TCP:
+                types[0] = "tcp";
+                break;
+        case GF_TRANSPORT_RDMA:
+                types[0] = "rdma";
+                break;
+        case GF_TRANSPORT_BOTH_TCP_RDMA:
+                types[0] = "tcp";
+                types[1] = "rdma";
+                break;
+        }
+}
+
+int
+generate_client_volfiles (glusterd_volinfo_t *volinfo,
+                          glusterd_client_type_t client_type)
+{
+        char               filepath[PATH_MAX] = {0,};
+        int                ret = -1;
+        char               *types[] = {NULL, NULL, NULL};
+        int                i = 0;
+        dict_t             *dict = NULL;
+        gf_transport_type  type = GF_TRANSPORT_TCP;
+
+        enumerate_transport_reqs (volinfo->transport_type, types);
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+        for (i = 0; types[i]; i++) {
+                memset (filepath, 0, sizeof (filepath));
+                ret = dict_set_str (dict, "client-transport-type", types[i]);
+                if (ret)
+                        goto out;
+                type = transport_str_to_type (types[i]);
+
+                ret = dict_set_uint32 (dict, "trusted-client", client_type);
+                if (ret)
+                        goto out;
+
+                if (client_type == GF_CLIENT_TRUSTED) {
+                        glusterd_get_trusted_client_filepath (filepath,
+                                                              volinfo,
+                                                              type);
+                } else {
+                        glusterd_get_client_filepath (filepath,
+                                                      volinfo,
+                                                      type);
+                }
+
+                ret = generate_single_transport_client_volfile (volinfo,
+                                                                filepath,
+                                                                dict);
+                if (ret)
+                        goto out;
+        }
+out:
+        if (dict)
+                dict_unref (dict);
+        return ret;
+}
+
+int
+glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
+                             glusterd_brickinfo_t *brickinfo)
+{
+        int ret = -1;
+
+        ret = glusterd_generate_brick_volfile (volinfo, brickinfo);
+        if (!ret)
+                ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
+        if (!ret)
+                ret = glusterd_fetchspec_notify (THIS);
+
+        return ret;
+}
+
+int
+glusterd_create_volfiles_and_notify_services (glusterd_volinfo_t *volinfo)
+{
+        int        ret  = -1;
+        xlator_t  *this = NULL;
+
+        this = THIS;
+
+        ret = generate_brick_volfiles (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not generate volfiles for bricks");
+                goto out;
+        }
+
+        ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not generate trusted client volfiles");
+                goto out;
+        }
+
+        ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not generate client volfiles");
+                goto out;
+        }
+
+        ret = glusterd_fetchspec_notify (this);
+
+out:
+        return ret;
+}
+
+int
+glusterd_create_global_volfile (int (*builder) (volgen_graph_t *graph,
+                                                dict_t *set_dict),
+                                char *filepath, dict_t  *mod_dict)
+{
+        volgen_graph_t graph = {0,};
+        int     ret = -1;
+
+        ret = builder (&graph, mod_dict);
+        if (!ret)
+                ret = volgen_write_volfile (&graph, filepath);
+
+        volgen_graph_free (&graph);
+
+        return ret;
+}
+
+int
+glusterd_create_nfs_volfile ()
+{
+        char            filepath[PATH_MAX] = {0,};
+        glusterd_conf_t *conf = THIS->private;
+
+        glusterd_get_nodesvc_volfile ("nfs", conf->workdir,
+                                            filepath, sizeof (filepath));
+        return glusterd_create_global_volfile (build_nfs_graph,
+                                               filepath, NULL);
+}
+
+int
+glusterd_create_shd_volfile ()
+{
+        char            filepath[PATH_MAX] = {0,};
+        int             ret = -1;
+        glusterd_conf_t *conf = THIS->private;
+        dict_t          *mod_dict = NULL;
+
+        mod_dict = dict_new ();
+        if (!mod_dict)
+                goto out;
+
+        ret = dict_set_uint32 (mod_dict, "cluster.background-self-heal-count", 0);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (mod_dict, "cluster.data-self-heal", "on");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (mod_dict, "cluster.metadata-self-heal", "on");
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (mod_dict, "cluster.entry-self-heal", "on");
+        if (ret)
+                goto out;
+
+        glusterd_get_nodesvc_volfile ("glustershd", conf->workdir,
+                                      filepath, sizeof (filepath));
+        ret = glusterd_create_global_volfile (build_shd_graph, filepath,
+                                              mod_dict);
+out:
+        if (mod_dict)
+                dict_unref (mod_dict);
+        return ret;
+}
+
+int
+glusterd_check_nfs_topology_identical (gf_boolean_t *identical)
+{
+        char            nfsvol[PATH_MAX]        = {0,};
+        char            tmpnfsvol[PATH_MAX]     = {0,};
+        glusterd_conf_t *conf                   = NULL;
+        xlator_t        *this                   = THIS;
+        int             ret                     = -1;
+        int             tmpclean                = 0;
+        int             tmpfd                   = -1;
+
+        if ((!identical) || (!this) || (!this->private))
+                goto out;
+
+        conf = (glusterd_conf_t *) this->private;
+
+        /* Fetch the original NFS volfile */
+        glusterd_get_nodesvc_volfile ("nfs", conf->workdir,
+                                      nfsvol, sizeof (nfsvol));
+
+        /* Create the temporary NFS volfile */
+        snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX");
+        tmpfd = mkstemp (tmpnfsvol);
+        if (tmpfd < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                                    "Unable to create temp file %s: (%s)",
+                                    tmpnfsvol, strerror (errno));
+                goto out;
+        }
+
+        tmpclean = 1; /* SET the flag to unlink() tmpfile */
+
+        ret = glusterd_create_global_volfile (build_nfs_graph,
+                                              tmpnfsvol, NULL);
+        if (ret)
+                goto out;
+
+        /* Compare the topology of volfiles */
+        ret = glusterd_check_topology_identical (nfsvol, tmpnfsvol,
+                                                 identical);
+out:
+        if (tmpfd >= 0)
+                close (tmpfd);
+        if (tmpclean)
+                unlink (tmpnfsvol);
+        return ret;
+}
+
+int
+glusterd_check_nfs_volfile_identical (gf_boolean_t *identical)
+{
+        char            nfsvol[PATH_MAX]        = {0,};
+        char            tmpnfsvol[PATH_MAX]     = {0,};
+        glusterd_conf_t *conf                   = NULL;
+        xlator_t        *this                   = NULL;
+        int             ret                     = -1;
+        int             need_unlink             = 0;
+        int             tmp_fd                  = -1;
+
+        this = THIS;
+
+        GF_ASSERT (this);
+        GF_ASSERT (identical);
+
+        conf = this->private;
+
+        glusterd_get_nodesvc_volfile ("nfs", conf->workdir,
+                                      nfsvol, sizeof (nfsvol));
+
+        snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX");
+
+        tmp_fd = mkstemp (tmpnfsvol);
+        if (tmp_fd < 0) {
+                gf_log ("", GF_LOG_WARNING, "Unable to create temp file %s: "
+                                "(%s)", tmpnfsvol, strerror (errno));
+                goto out;
+        }
+
+        need_unlink = 1;
+
+        ret = glusterd_create_global_volfile (build_nfs_graph,
+                                              tmpnfsvol, NULL);
+        if (ret)
+                goto out;
+
+        ret = glusterd_check_files_identical (nfsvol, tmpnfsvol,
+                                              identical);
+        if (ret)
+                goto out;
+
+out:
+        if (need_unlink)
+                unlink (tmpnfsvol);
+
+        if (tmp_fd >= 0)
+                close (tmp_fd);
+
+        return ret;
+}
+
+int
+glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo)
+{
+        int  ret = 0;
+        char filename[PATH_MAX] = {0,};
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (brickinfo);
+
+        get_brick_filepath (filename, volinfo, brickinfo);
+        ret = unlink (filename);
+        if (ret)
+                gf_log ("glusterd", GF_LOG_ERROR, "failed to delete file: %s, "
+                        "reason: %s", filename, strerror (errno));
+        return ret;
+}
+
+int
+validate_shdopts (glusterd_volinfo_t *volinfo,
+                  dict_t *val_dict,
+                  char **op_errstr)
+{
+        volgen_graph_t graph = {0,};
+        int     ret = -1;
+
+        graph.errstr = op_errstr;
+
+        if (!glusterd_is_volume_replicate (volinfo)) {
+                ret = 0;
+                goto out;
+        }
+        ret = dict_set_str (val_dict, "graph-check", "on");
+        if (ret)
+                goto out;
+        ret = build_shd_graph (&graph, val_dict);
+        if (!ret)
+                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+
+        volgen_graph_free (&graph);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+out:
+        dict_del (val_dict, "graph-check");
+        return ret;
+}
+
+int
+validate_nfsopts (glusterd_volinfo_t *volinfo,
+                    dict_t *val_dict,
+                    char **op_errstr)
+{
+        volgen_graph_t graph = {0,};
+        int     ret = -1;
+        char    transport_type[16] = {0,};
+        char    *tt                = NULL;
+        char    err_str[4096]      = {0,};
+        xlator_t *this             = THIS;
+
+        GF_ASSERT (this);
+
+        graph.errstr = op_errstr;
+
+        get_vol_transport_type (volinfo, transport_type);
+        ret = dict_get_str (val_dict, "nfs.transport-type", &tt);
+        if (!ret) {
+                if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
+                        snprintf (err_str, sizeof (err_str), "Changing nfs "
+                                  "transport type is allowed only for volumes "
+                                  "of transport type tcp,rdma");
+                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                        *op_errstr = gf_strdup (err_str);
+                        ret = -1;
+                        goto out;
+                }
+                if (strcmp (tt,"tcp") && strcmp (tt,"rdma")) {
+                        snprintf (err_str, sizeof (err_str), "wrong transport "
+                                  "type %s", tt);
+                        *op_errstr = gf_strdup (err_str);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = dict_set_str (val_dict, "volume-name", volinfo->volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set volume name");
+                goto out;
+        }
+
+        ret = build_nfs_graph (&graph, val_dict);
+        if (!ret)
+                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+
+        volgen_graph_free (&graph);
+
+out:
+        if (dict_get (val_dict, "volume-name"))
+                dict_del (val_dict, "volume-name");
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+
+int
+validate_clientopts (glusterd_volinfo_t *volinfo,
+                     dict_t *val_dict,
+                     char **op_errstr)
+{
+        volgen_graph_t graph = {0,};
+        int            ret   = -1;
+
+        GF_ASSERT (volinfo);
+
+        graph.errstr = op_errstr;
+
+        ret = build_client_graph (&graph, volinfo, val_dict);
+        if (!ret)
+                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+
+        volgen_graph_free (&graph);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+validate_brickopts (glusterd_volinfo_t *volinfo,
+                    glusterd_brickinfo_t *brickinfo,
+                    dict_t *val_dict,
+                    char **op_errstr)
+{
+        volgen_graph_t graph = {0,};
+        int            ret   = -1;
+
+        GF_ASSERT (volinfo);
+
+        graph.errstr = op_errstr;
+
+        ret = build_server_graph (&graph, volinfo, val_dict, brickinfo);
+        if (!ret)
+                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+
+        volgen_graph_free (&graph);
+
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_validate_brickreconf (glusterd_volinfo_t *volinfo,
+                               dict_t *val_dict,
+                               char **op_errstr)
+{
+        glusterd_brickinfo_t *brickinfo = NULL;
+        int                   ret = -1;
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "Validating %s", brickinfo->hostname);
+
+                ret = validate_brickopts (volinfo, brickinfo, val_dict,
+                                          op_errstr);
+                if (ret)
+                        goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static int
+_check_globalopt (dict_t *this, char *key, data_t *value, void *ret_val)
+{
+        int *ret = NULL;
+
+        ret = ret_val;
+        if (*ret)
+                return 0;
+        if (!glusterd_check_globaloption (key))
+                *ret = 1;
+
+        return 0;
+}
+
+int
+glusterd_validate_globalopts (glusterd_volinfo_t *volinfo,
+                              dict_t *val_dict, char **op_errstr)
+{
+        int ret = 0;
+
+        dict_foreach (val_dict, _check_globalopt, &ret);
+        if (ret) {
+                *op_errstr = gf_strdup ( "option specified is not a global option");
+                return -1;
+        }
+        ret = glusterd_validate_brickreconf (volinfo, val_dict, op_errstr);
+
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "Could not Validate  bricks");
+                goto out;
+        }
+
+        ret = validate_clientopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "Could not Validate client");
+                goto out;
+        }
+
+        ret = validate_nfsopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate nfs");
+                goto out;
+        }
+
+        ret = validate_shdopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate self-heald");
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static int
+_check_localopt (dict_t *this, char *key, data_t *value, void *ret_val)
+{
+        int *ret = NULL;
+
+        ret = ret_val;
+        if (*ret)
+                return 0;
+        if (!glusterd_check_localoption (key))
+                *ret = 1;
+
+        return 0;
+}
+
+int
+glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict,
+                              char **op_errstr)
+{
+        int ret = 0;
+
+        dict_foreach (val_dict, _check_localopt, &ret);
+        if (ret) {
+                *op_errstr = gf_strdup ( "option specified is not a local option");
+                return -1;
+        }
+        ret = glusterd_validate_brickreconf (volinfo, val_dict, op_errstr);
+
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "Could not Validate  bricks");
+                goto out;
+        }
+
+        ret = validate_clientopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG,
+                        "Could not Validate client");
+                goto out;
+        }
+
+        ret = validate_nfsopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate nfs");
+                goto out;
+        }
+
+
+        ret = validate_shdopts (volinfo, val_dict, op_errstr);
+        if (ret) {
+                gf_log ("", GF_LOG_DEBUG, "Could not Validate self-heald");
+                goto out;
+        }
+
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static struct volopt_map_entry *
+_gd_get_vmep (char *key) {
+        char *completion = NULL;
+        struct volopt_map_entry *vmep = NULL;
+        int ret = 0;
+
+        COMPLETE_OPTION ((char *)key, completion, ret);
+        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+                if (strcmp (vmep->key, key) == 0)
+                        return vmep;
+        }
+
+        return NULL;
+}
+
+uint32_t
+glusterd_get_op_version_for_key (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep)
+                return vmep->op_version;
+
+        return 0;
+}
+
+gf_boolean_t
+gd_is_client_option (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep && (vmep->flags & OPT_FLAG_CLIENT_OPT))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+gf_boolean_t
+gd_is_xlator_option (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+        if (vmep && (vmep->flags & OPT_FLAG_XLATOR_OPT))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+volume_option_type_t
+_gd_get_option_type (char *key)
+{
+        struct volopt_map_entry *vmep = NULL;
+        void                    *dl_handle = NULL;
+        volume_opt_list_t       vol_opt_list = {{0},};
+        int                     ret = -1;
+        volume_option_t         *opt = NULL;
+        char                    *xlopt_key = NULL;
+        volume_option_type_t    opt_type = GF_OPTION_TYPE_MAX;
+
+        GF_ASSERT (key);
+
+        vmep = _gd_get_vmep (key);
+
+        if (vmep) {
+                INIT_LIST_HEAD (&vol_opt_list.list);
+                ret = xlator_volopt_dynload (vmep->voltype, &dl_handle,
+                                             &vol_opt_list);
+                if (ret)
+                        goto out;
+
+                if (_get_xlator_opt_key_from_vme (vmep, &xlopt_key))
+                        goto out;
+
+                opt = xlator_volume_option_get_list (&vol_opt_list, xlopt_key);
+                _free_xlator_opt_key (xlopt_key);
+
+                if (opt)
+                        opt_type = opt->type;
+        }
+
+out:
+        if (dl_handle) {
+                dlclose (dl_handle);
+                dl_handle = NULL;
+        }
+
+        return opt_type;
+}
+
+gf_boolean_t
+gd_is_boolean_option (char *key)
+{
+        GF_ASSERT (key);
+
+        if (GF_OPTION_TYPE_BOOL == _gd_get_option_type (key))
+                return _gf_true;
+
+        return _gf_false;
+}
+
+/* This function will restore origin volume to it's snap.
+ * The restore operation will simply replace the Gluster origin
+ * volume with the snap volume.
+ * TODO: Multi-volume delete to be done.
+ *       Cleanup in case of restore failure is pending.
+ *
+ * @param orig_vol      volinfo of origin volume
+ * @param snap_vol      volinfo of snapshot volume
+ *
+ * @return 0 on success and negative value on error
+ */
+int
+gd_restore_snap_volume (dict_t *rsp_dict,
+                        glusterd_volinfo_t *orig_vol,
+                        glusterd_volinfo_t *snap_vol)
+{
+        int                     ret             = -1;
+        glusterd_volinfo_t      *new_volinfo    = NULL;
+        glusterd_snap_t         *snap           = NULL;
+        xlator_t                *this           = NULL;
+        glusterd_conf_t         *conf           = NULL;
+        glusterd_volinfo_t      *temp_volinfo   = NULL;
+        glusterd_volinfo_t      *voliter        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (rsp_dict);
+        conf = this->private;
+        GF_ASSERT (conf);
+
+        GF_VALIDATE_OR_GOTO (this->name, orig_vol, out);
+        GF_VALIDATE_OR_GOTO (this->name, snap_vol, out);
+        snap = snap_vol->snapshot;
+        GF_VALIDATE_OR_GOTO (this->name, snap, out);
+
+        /* Snap volume must be stoped before performing the
+         * restore operation.
+         */
+        ret = glusterd_stop_volume (snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
+                        "snap volume");
+                goto out;
+        }
+
+        /* Create a new volinfo for the restored volume */
+        ret = glusterd_volinfo_dup (snap_vol, &new_volinfo, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create volinfo");
+                goto out;
+        }
+
+        /* Following entries need to be derived from origin volume. */
+        strcpy (new_volinfo->volname, orig_vol->volname);
+        uuid_copy (new_volinfo->volume_id, orig_vol->volume_id);
+        new_volinfo->snap_count = orig_vol->snap_count;
+        new_volinfo->snap_max_hard_limit = orig_vol->snap_max_hard_limit;
+        new_volinfo->is_volume_restored = _gf_true;
+
+        /* Bump the version of the restored volume, so that nodes *
+         * which are done can sync during handshake */
+        new_volinfo->version = orig_vol->version;
+
+        list_for_each_entry_safe (voliter, temp_volinfo,
+                         &orig_vol->snap_volumes, snapvol_list) {
+                list_add_tail (&voliter->snapvol_list,
+                               &new_volinfo->snap_volumes);
+        }
+        /* Copy the snap vol info to the new_volinfo.*/
+        ret = glusterd_snap_volinfo_restore (rsp_dict, new_volinfo, snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to restore snap");
+                (void)glusterd_volinfo_delete (new_volinfo);
+                goto out;
+        }
+
+        /* If the orig_vol is already restored then we should delete
+         * the backend LVMs */
+        if (orig_vol->is_volume_restored) {
+                ret = glusterd_lvm_snapshot_remove (rsp_dict, orig_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to remove "
+                                "LVM backend");
+                        (void)glusterd_volinfo_delete (new_volinfo);
+                        goto out;
+                }
+        }
+
+        /* Once the new_volinfo is completely constructed then delete
+         * the orinal volinfo
+         */
+        ret = glusterd_volinfo_delete (orig_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to delete volinfo");
+                (void)glusterd_volinfo_delete (new_volinfo);
+                goto out;
+        }
+
+        /* New volinfo always shows the status as created. Therefore
+         * set the status to stop. */
+        glusterd_set_volume_status (new_volinfo, GLUSTERD_STATUS_STOPPED);
+
+        list_add_tail (&new_volinfo->vol_list, &conf->volumes);
+
+        /* Now delete the snap entry. As a first step delete the snap
+         * volume information stored in store. */
+        ret = glusterd_snap_remove (rsp_dict, snap, _gf_false, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
+                        "snap %s", snap->snapname);
+                goto out;
+        }
+
+        ret = glusterd_store_volinfo (new_volinfo,
+                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo");
+                goto out;
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
new file mode 100644
index 000000000..fcbaaf93e
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -0,0 +1,176 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_VOLGEN_H_
+#define _GLUSTERD_VOLGEN_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterd.h"
+
+/* volopt map key name definitions */
+
+#define VKEY_DIAG_CNT_FOP_HITS    "diagnostics.count-fop-hits"
+#define VKEY_DIAG_LAT_MEASUREMENT "diagnostics.latency-measurement"
+#define VKEY_FEATURES_LIMIT_USAGE "features.limit-usage"
+#define VKEY_MARKER_XTIME         GEOREP".indexing"
+#define VKEY_MARKER_XTIME_FORCE   GEOREP".ignore-pid-check"
+#define VKEY_CHANGELOG            "changelog.changelog"
+#define VKEY_FEATURES_QUOTA       "features.quota"
+
+#define AUTH_ALLOW_MAP_KEY "auth.allow"
+#define AUTH_REJECT_MAP_KEY "auth.reject"
+#define NFS_DISABLE_MAP_KEY "nfs.disable"
+#define AUTH_ALLOW_OPT_KEY "auth.addr.*.allow"
+#define AUTH_REJECT_OPT_KEY "auth.addr.*.reject"
+#define NFS_DISABLE_OPT_KEY "nfs.*.disable"
+
+
+typedef enum {
+        GF_CLIENT_TRUSTED,
+        GF_CLIENT_OTHER
+} glusterd_client_type_t;
+
+#define COMPLETE_OPTION(key, completion, ret)                           \
+        do {                                                            \
+                if (!strchr (key, '.')) {                               \
+                        ret = option_complete (key, &completion);       \
+                        if (ret) {                                      \
+                                gf_log ("", GF_LOG_ERROR, "Out of memory"); \
+                                return _gf_false;                       \
+                        }                                               \
+                                                                        \
+                        if (!completion) {                              \
+                                gf_log ("", GF_LOG_ERROR, "option %s does not" \
+                                        "exist", key);                  \
+                                return _gf_false;                       \
+                        }                                               \
+                }                                                       \
+                                                                        \
+                if (completion)                                         \
+                        GF_FREE (completion);                           \
+        } while (0);
+
+typedef enum gd_volopt_flags_ {
+        OPT_FLAG_NONE,
+        OPT_FLAG_FORCE = 0x01,      // option needs force to be reset
+        OPT_FLAG_XLATOR_OPT = 0x02, // option enables/disables xlators
+        OPT_FLAG_CLIENT_OPT = 0x04, // option affects clients
+} gd_volopt_flags_t;
+
+typedef enum {
+        GF_XLATOR_POSIX = 0,
+        GF_XLATOR_ACL,
+        GF_XLATOR_LOCKS,
+        GF_XLATOR_IOT,
+        GF_XLATOR_INDEX,
+        GF_XLATOR_MARKER,
+        GF_XLATOR_IO_STATS,
+        GF_XLATOR_BD,
+        GF_XLATOR_NONE,
+} glusterd_server_xlator_t;
+
+/* As of now debug xlators can be loaded only below fuse in the client
+ * graph via cli. More xlators can be added below when the cli option
+ * for adding debug xlators anywhere in the client graph has to be made
+ * available.
+ */
+typedef enum {
+        GF_CLNT_XLATOR_FUSE = 0,
+        GF_CLNT_XLATOR_NONE,
+} glusterd_client_xlator_t;
+
+typedef enum  { DOC, NO_DOC, GLOBAL_DOC, GLOBAL_NO_DOC } option_type_t;
+
+typedef int (*vme_option_validation) (dict_t *dict, char *key, char *value,
+                                      char **op_errstr);
+
+struct volopt_map_entry {
+        char *key;
+        char *voltype;
+        char *option;
+        char *value;
+        option_type_t type;
+        uint32_t flags;
+        uint32_t op_version;
+        char *description;
+        vme_option_validation validate_fn;
+        /* If client_option is true, the option affects clients.
+         * this is used to calculate client-op-version of volumes
+         */
+        //gf_boolean_t client_option;
+};
+
+int glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
+                                 glusterd_brickinfo_t *brickinfo);
+
+int glusterd_create_volfiles_and_notify_services (glusterd_volinfo_t *volinfo);
+
+void glusterd_get_nfs_filepath (char *filename);
+
+void glusterd_get_shd_filepath (char *filename);
+
+int glusterd_create_nfs_volfile ();
+int glusterd_create_shd_volfile ();
+
+int glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
+                             glusterd_brickinfo_t *brickinfo);
+int
+glusterd_delete_snap_volfile (glusterd_volinfo_t *volinfo,
+                              glusterd_volinfo_t *snap_volinfo,
+                              glusterd_brickinfo_t *brickinfo);
+
+int glusterd_volinfo_get (glusterd_volinfo_t *volinfo, char *key, char **value);
+int glusterd_volinfo_get_boolean (glusterd_volinfo_t *volinfo, char *key);
+
+int glusterd_validate_globalopts (glusterd_volinfo_t *volinfo, dict_t *val_dict, char **op_errstr);
+
+int glusterd_validate_localopts (dict_t *val_dict, char **op_errstr);
+gf_boolean_t glusterd_check_globaloption (char *key);
+gf_boolean_t
+glusterd_check_voloption_flags (char *key, int32_t flags);
+gf_boolean_t
+glusterd_is_valid_volfpath (char *volname, char *brick);
+int generate_brick_volfiles (glusterd_volinfo_t *volinfo);
+int generate_snap_brick_volfiles (glusterd_volinfo_t *volinfo,
+                                  glusterd_volinfo_t *snap_volinfo);
+int generate_client_volfiles (glusterd_volinfo_t *volinfo,
+                              glusterd_client_type_t client_type);
+int
+generate_snap_client_volfiles (glusterd_volinfo_t *actual_volinfo,
+                               glusterd_volinfo_t *snap_volinfo,
+                               glusterd_client_type_t client_type,
+                               gf_boolean_t vol_restore);
+int glusterd_get_volopt_content (dict_t *dict, gf_boolean_t xml_out);
+char*
+glusterd_get_trans_type_rb (gf_transport_type ttype);
+int
+glusterd_check_nfs_volfile_identical (gf_boolean_t *identical);
+int
+glusterd_check_nfs_topology_identical (gf_boolean_t *identical);
+
+uint32_t
+glusterd_get_op_version_for_key (char *key);
+
+gf_boolean_t
+gd_is_client_option (char *key);
+
+gf_boolean_t
+gd_is_xlator_option (char *key);
+
+gf_boolean_t
+gd_is_boolean_option (char *key);
+
+int gd_restore_snap_volume (dict_t *rsp_dict,
+                            glusterd_volinfo_t *orig_vol,
+                            glusterd_volinfo_t *snap_vol);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
new file mode 100644
index 000000000..0d322b9ad
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -0,0 +1,2225 @@
+/*
+   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#ifdef HAVE_BD_XLATOR
+#include <lvm2app.h>
+#endif
+
+#include "common-utils.h"
+#include "syscall.h"
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "run.h"
+
+#define glusterd_op_start_volume_args_get(dict, volname, flags) \
+        glusterd_op_stop_volume_args_get (dict, volname, flags)
+
+
+int
+__glusterd_handle_create_volume (rpcsvc_request_t *req)
+{
+        int32_t                 ret         = -1;
+        gf_cli_req              cli_req     = {{0,}};
+        dict_t                 *dict        = NULL;
+        char                   *bricks      = NULL;
+        char                   *volname     = NULL;
+        int                    brick_count  = 0;
+        void                   *cli_rsp     = NULL;
+        char                    err_str[2048] = {0,};
+        gf_cli_rsp              rsp         = {0,};
+        xlator_t               *this        = NULL;
+        char                   *free_ptr    = NULL;
+        char                   *trans_type  = NULL;
+        uuid_t                  volume_id   = {0,};
+        uuid_t                  tmp_uuid    = {0};
+        int32_t                 type        = 0;
+        char                   *username    = NULL;
+        char                   *password    = NULL;
+
+        GF_ASSERT (req);
+
+        this = THIS;
+        GF_ASSERT(this);
+
+        ret = -1;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                snprintf (err_str, sizeof (err_str), "Failed to decode request "
+                          "received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received create volume req");
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        if ((ret = glusterd_check_volume_exists (volname))) {
+                snprintf (err_str, sizeof (err_str), "Volume %s already exists",
+                          volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &brick_count);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get brick count"
+                          " for volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "type", &type);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get type of "
+                          "volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "transport", &trans_type);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get "
+                          "transport-type of volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+        ret = dict_get_str (dict, "bricks", &bricks);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get bricks for "
+                          "volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        if (!dict_get (dict, "force")) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get 'force' flag");
+                goto out;
+        }
+
+        uuid_generate (volume_id);
+        free_ptr = gf_strdup (uuid_utoa (volume_id));
+        ret = dict_set_dynstr (dict, "volume-id", free_ptr);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to set volume "
+                          "id of volume %s", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+        free_ptr = NULL;
+
+        /* generate internal username and password */
+
+        uuid_generate (tmp_uuid);
+        username = gf_strdup (uuid_utoa (tmp_uuid));
+        ret = dict_set_dynstr (dict, "internal-username", username);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set username for "
+                        "volume %s", volname);
+                goto out;
+        }
+
+        uuid_generate (tmp_uuid);
+        password = gf_strdup (uuid_utoa (tmp_uuid));
+        ret = dict_set_dynstr (dict, "internal-password", password);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set password for "
+                        "volume %s", volname);
+                goto out;
+        }
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_CREATE_VOLUME, dict);
+
+out:
+        if (ret) {
+                rsp.op_ret = -1;
+                rsp.op_errno = 0;
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                rsp.op_errstr = err_str;
+                cli_rsp = &rsp;
+                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
+                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
+                ret = 0; //Client response sent, prevent second response
+        }
+
+        GF_FREE(free_ptr);
+
+        return ret;
+}
+
+int
+glusterd_handle_create_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_create_volume);
+}
+
+int
+__glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        char                            *volname = NULL;
+        dict_t                          *dict = NULL;
+        glusterd_op_t                   cli_op = GD_OP_START_VOLUME;
+        char                            errstr[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (errstr, sizeof (errstr), "Failed to decode message "
+                        "received from cli");
+                req->rpc_err = GARBAGE_ARGS;
+                gf_log (this->name, sizeof (errstr), "%s", errstr);
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (errstr, sizeof (errstr), "Unable to decode "
+                                  "the command");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (errstr, sizeof (errstr), "Unable to get volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received start vol req"
+                " for volume %s", volname);
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_START_VOLUME, dict);
+
+out:
+        free (cli_req.dict.dict_val); //its malloced by xdr
+
+        if (ret) {
+                if(errstr[0] == '\0')
+                        snprintf (errstr, sizeof (errstr), "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, errstr);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_start_volume);
+}
+
+int
+__glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        char                            *dup_volname = NULL;
+        dict_t                          *dict = NULL;
+        glusterd_op_t                   cli_op = GD_OP_STOP_VOLUME;
+        xlator_t                        *this = NULL;
+        char                            err_str[2048] = {0,};
+
+        this = THIS;
+        GF_ASSERT (this);
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode message "
+                          "received from cli");
+                req->rpc_err = GARBAGE_ARGS;
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &dup_volname);
+
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received stop vol req "
+                "for volume %s", dup_volname);
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_STOP_VOLUME, dict);
+
+out:
+        free (cli_req.dict.dict_val); //its malloced by xdr
+
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_stop_volume);
+}
+
+int
+__glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+{
+        int32_t        ret         = -1;
+        gf_cli_req     cli_req     = {{0,},};
+        glusterd_op_t  cli_op      = GD_OP_DELETE_VOLUME;
+        dict_t        *dict        = NULL;
+        char          *volname     = NULL;
+        char          err_str[2048]= {0,};
+        xlator_t      *this        = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                snprintf (err_str, sizeof (err_str), "Failed to decode request "
+                          "received from cli");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to decode "
+                                  "the command");
+                        goto out;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Failed to get volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "Received delete vol req"
+                "for volume %s", volname);
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_DELETE_VOLUME, dict);
+
+out:
+        free (cli_req.dict.dict_val); //its malloced by xdr
+
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_delete_volume);
+}
+
+int
+__glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        dict_t                          *dict = NULL;
+        glusterd_op_t                   cli_op = GD_OP_HEAL_VOLUME;
+        char                            *volname = NULL;
+        glusterd_volinfo_t              *volinfo = NULL;
+        xlator_t                        *this = NULL;
+        char                            op_errstr[2048] = {0,};
+
+        GF_ASSERT (req);
+
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (op_errstr, sizeof (op_errstr),
+                        "Unable to decode the command");
+                        goto out;
+                } else {
+                        dict->extra_stdfree = cli_req.dict.dict_val;
+                }
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (op_errstr, sizeof (op_errstr), "Unable to find "
+                          "volume name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Received heal vol req "
+                "for volume %s", volname);
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (op_errstr, sizeof (op_errstr),
+                          "Volume %s does not exist", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
+                goto out;
+        }
+
+        ret = glusterd_add_bricks_hname_path_to_dict (dict, volinfo);
+        if (ret)
+                goto out;
+
+        ret = dict_set_int32 (dict, "count", volinfo->brick_count);
+        if (ret)
+                goto out;
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_HEAL_VOLUME, dict);
+
+out:
+        if (ret) {
+                if (op_errstr[0] == '\0')
+                        snprintf (op_errstr, sizeof (op_errstr),
+                                  "operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, op_errstr);
+        }
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_heal_volume);
+}
+
+int
+__glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
+{
+        int32_t                         ret = -1;
+        gf_cli_req                      cli_req = {{0,}};
+        char                            *volname = NULL;
+        char                            *options = NULL;
+        dict_t                          *dict = NULL;
+        int32_t                         option_cnt = 0;
+        glusterd_op_t                   cli_op = GD_OP_STATEDUMP_VOLUME;
+        char                            err_str[2048] = {0,};
+        xlator_t                        *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (req);
+
+        ret = -1;
+        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        if (ret < 0) {
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
+        if (cli_req.dict.dict_len) {
+                /* Unserialize the dictionary */
+                dict  = dict_new ();
+
+                ret = dict_unserialize (cli_req.dict.dict_val,
+                                        cli_req.dict.dict_len,
+                                        &dict);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to "
+                                "unserialize req-buffer to dictionary");
+                        snprintf (err_str, sizeof (err_str), "Unable to "
+                                  "decode the command");
+                        goto out;
+                }
+        }
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get the volume "
+                          "name");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "options", &options);
+        if (ret) {
+                snprintf (err_str, sizeof (err_str), "Unable to get options");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "option_cnt", &option_cnt);
+        if (ret) {
+                snprintf (err_str , sizeof (err_str), "Unable to get option "
+                          "count");
+                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
+                goto out;
+        }
+
+
+        gf_log (this->name, GF_LOG_INFO, "Received statedump request for "
+                "volume %s with options %s", volname, options);
+
+        ret = glusterd_op_begin_synctask (req, GD_OP_STATEDUMP_VOLUME, dict);
+
+out:
+        if (ret) {
+                if (err_str[0] == '\0')
+                        snprintf (err_str, sizeof (err_str),
+                                  "Operation failed");
+                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
+                                                     dict, err_str);
+        }
+        free (cli_req.dict.dict_val);
+
+        return ret;
+}
+
+int
+glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
+{
+        return glusterd_big_locked_handler (req,
+                                            __glusterd_handle_cli_statedump_volume);
+}
+
+#ifdef HAVE_BD_XLATOR
+/*
+ * Validates if given VG in the brick exists or not. Also checks if VG has
+ * GF_XATTR_VOL_ID_KEY tag set to avoid using same VG for multiple bricks.
+ * Tag is checked only during glusterd_op_stage_create_volume. Tag is set during
+ * glusterd_validate_and_create_brickpath().
+ * @brick - brick info, @check_tag - check for VG tag or not
+ * @msg - Error message to return to caller
+ */
+int
+glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg)
+{
+        lvm_t                     handle     = NULL;
+        vg_t                      vg         = NULL;
+        char                     *vg_name    = NULL;
+        int                       retval     = 0;
+        char                     *p          = NULL;
+        char                     *ptr        = NULL;
+        struct dm_list           *dm_lvlist = NULL;
+        struct dm_list           *dm_seglist = NULL;
+        struct lvm_lv_list       *lv_list    = NULL;
+        struct lvm_property_value prop       = {0, };
+        struct lvm_lvseg_list    *seglist    = NULL;
+        struct dm_list           *taglist    = NULL;
+        struct lvm_str_list      *strl       = NULL;
+
+        handle = lvm_init (NULL);
+        if (!handle) {
+                sprintf (msg, "lvm_init failed, could not validate vg");
+                return -1;
+        }
+        if (*brick->vg == '\0') { /* BD xlator has vg in brick->path */
+                p = gf_strdup (brick->path);
+                vg_name = strtok_r (p, "/", &ptr);
+        } else
+                vg_name = brick->vg;
+
+        vg = lvm_vg_open (handle, vg_name, "r", 0);
+        if (!vg) {
+                sprintf (msg, "no such vg: %s", vg_name);
+                retval = -1;
+                goto out;
+        }
+        if (!check_tag)
+                goto next;
+
+        taglist = lvm_vg_get_tags (vg);
+        if (!taglist)
+                goto next;
+
+        dm_list_iterate_items (strl, taglist) {
+                if (!strncmp(strl->str, GF_XATTR_VOL_ID_KEY,
+                             strlen (GF_XATTR_VOL_ID_KEY))) {
+                            sprintf (msg, "VG %s is already part of"
+                                    " a brick", vg_name);
+                            retval = -1;
+                            goto out;
+                    }
+        }
+next:
+
+        brick->caps = CAPS_BD | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+
+        dm_lvlist = lvm_vg_list_lvs (vg);
+        if (!dm_lvlist)
+                goto out;
+
+        dm_list_iterate_items (lv_list, dm_lvlist) {
+                dm_seglist = lvm_lv_list_lvsegs (lv_list->lv);
+                dm_list_iterate_items (seglist, dm_seglist) {
+                        prop = lvm_lvseg_get_property (seglist->lvseg,
+                                                       "segtype");
+                        if (!prop.is_valid || !prop.value.string)
+                                continue;
+                        if (!strcmp (prop.value.string, "thin-pool")) {
+                                brick->caps |= CAPS_THIN;
+                                gf_log (THIS->name, GF_LOG_INFO, "Thin Pool "
+                                        "\"%s\" will be used for thin LVs",
+                                        lvm_lv_get_name (lv_list->lv));
+                                break;
+                        }
+                }
+        }
+
+        retval = 0;
+out:
+        if (vg)
+                lvm_vg_close (vg);
+        lvm_quit (handle);
+        if (p)
+                GF_FREE (p);
+        return retval;
+}
+#endif
+
+/* op-sm */
+int
+glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = 0;
+        char                                    *volname = NULL;
+        gf_boolean_t                            exists = _gf_false;
+        char                                    *bricks = NULL;
+        char                                    *brick_list = NULL;
+        char                                    *free_ptr = NULL;
+        glusterd_brickinfo_t                    *brick_info = NULL;
+        int32_t                                 brick_count = 0;
+        int32_t                                 i = 0;
+        char                                    *brick = NULL;
+        char                                    *tmpptr = NULL;
+        xlator_t                                *this = NULL;
+        glusterd_conf_t                         *priv = NULL;
+        char                                    msg[2048] = {0};
+        uuid_t                                  volume_uuid;
+        char                                    *volume_uuid_str;
+        gf_boolean_t                             is_force = _gf_false;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        if (exists) {
+                snprintf (msg, sizeof (msg), "Volume %s already exists",
+                          volname);
+                ret = -1;
+                goto out;
+        } else {
+                ret = 0;
+        }
+
+        ret = dict_get_int32 (dict, "count", &brick_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count "
+                        "for volume %s", volname);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volume-id", &volume_uuid_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume id of "
+                        "volume %s", volname);
+                goto out;
+        }
+
+        ret = uuid_parse (volume_uuid_str, volume_uuid);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to parse volume id of"
+                        " volume %s", volname);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "bricks", &bricks);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks for "
+                        "volume %s", volname);
+                goto out;
+        }
+
+        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+
+        if (bricks) {
+                brick_list = gf_strdup (bricks);
+                if (!brick_list) {
+                        ret = -1;
+                        goto out;
+                } else {
+                        free_ptr = brick_list;
+                }
+        }
+
+        while ( i < brick_count) {
+                i++;
+                brick= strtok_r (brick_list, " \n", &tmpptr);
+                brick_list = tmpptr;
+
+                if (!glusterd_store_is_valid_brickpath (volname, brick) ||
+                        !glusterd_is_valid_volfpath (volname, brick)) {
+                        snprintf (msg, sizeof (msg), "brick path %s is too "
+                                  "long.", brick);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = glusterd_brickinfo_new_from_brick (brick, &brick_info);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_new_brick_validate (brick, brick_info, msg,
+                                                   sizeof (msg));
+                if (ret)
+                        goto out;
+
+                ret = glusterd_resolve_brick (brick_info);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brick_info->hostname, brick_info->path);
+                        goto out;
+                }
+
+                if (!uuid_compare (brick_info->uuid, MY_UUID)) {
+
+#ifdef HAVE_BD_XLATOR
+                        if (brick_info->vg[0]) {
+                                ret = glusterd_is_valid_vg (brick_info, 1, msg);
+                                if (ret)
+                                        goto out;
+                        }
+#endif
+                        ret = glusterd_validate_and_create_brickpath (brick_info,
+                                                          volume_uuid, op_errstr,
+                                                          is_force);
+                        if (ret)
+                                goto out;
+                        brick_list = tmpptr;
+                }
+                glusterd_brickinfo_delete (brick_info);
+                brick_info = NULL;
+        }
+out:
+        GF_FREE (free_ptr);
+        if (brick_info)
+                glusterd_brickinfo_delete (brick_info);
+
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_op_stop_volume_args_get (dict_t *dict, char** volname, int *flags)
+{
+        int ret = -1;
+        xlator_t *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        if (!dict || !volname || !flags)
+                goto out;
+
+        ret = dict_get_str (dict, "volname", volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "flags", flags);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get flags");
+                goto out;
+        }
+out:
+        return ret;
+}
+
+int
+glusterd_op_statedump_volume_args_get (dict_t *dict, char **volname,
+                                       char **options, int *option_cnt)
+{
+        int ret = -1;
+
+        if (!dict || !volname || !options || !option_cnt)
+                goto out;
+
+        ret = dict_get_str (dict, "volname", volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volname");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "options", options);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get options");
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "option_cnt", option_cnt);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get option count");
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = 0;
+        char                                    *volname = NULL;
+        int                                     flags = 0;
+        gf_boolean_t                            exists = _gf_false;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        glusterd_brickinfo_t                    *brickinfo = NULL;
+        char                                    msg[2048];
+        glusterd_conf_t                         *priv = NULL;
+        xlator_t                                *this = NULL;
+        uuid_t                                  volume_id = {0,};
+        char                                    volid[50] = {0,};
+        char                                    xattr_volid[50] = {0,};
+        int                                     caps = 0;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = glusterd_op_start_volume_args_get (dict, &volname, &flags);
+        if (ret)
+                goto out;
+
+        exists = glusterd_check_volume_exists (volname);
+
+        if (!exists) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                ret = -1;
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+                if (glusterd_is_volume_started (volinfo)) {
+                        snprintf (msg, sizeof (msg), "Volume %s already "
+                                  "started", volname);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brickinfo->hostname, brickinfo->path);
+                        goto out;
+                }
+
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                ret = gf_lstat_dir (brickinfo->path, NULL);
+                if (ret && (flags & GF_CLI_FLAG_OP_FORCE)) {
+                        continue;
+                } else if (ret) {
+                        snprintf (msg, sizeof (msg), "Failed to find "
+                                          "brick directory %s for volume %s. "
+                                          "Reason : %s", brickinfo->path,
+                                          volname, strerror (errno));
+                        goto out;
+                }
+                ret = sys_lgetxattr (brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                                     volume_id, 16);
+                if (ret < 0 && (!(flags & GF_CLI_FLAG_OP_FORCE))) {
+                        snprintf (msg, sizeof (msg), "Failed to get "
+                                  "extended attribute %s for brick dir %s. "
+                                  "Reason : %s", GF_XATTR_VOL_ID_KEY,
+                                  brickinfo->path, strerror (errno));
+                        ret = -1;
+                        goto out;
+                } else if (ret < 0) {
+                        ret = sys_lsetxattr (brickinfo->path,
+                                             GF_XATTR_VOL_ID_KEY,
+                                             volinfo->volume_id, 16,
+                                             XATTR_CREATE);
+                        if (ret) {
+                                snprintf (msg, sizeof (msg), "Failed to set "
+                                        "extended attribute %s on %s. Reason: "
+                                        "%s", GF_XATTR_VOL_ID_KEY,
+                                        brickinfo->path, strerror (errno));
+                                goto out;
+                        } else {
+                                continue;
+                        }
+                }
+                if (uuid_compare (volinfo->volume_id, volume_id)) {
+                        snprintf (msg, sizeof (msg), "Volume id mismatch for "
+                                  "brick %s:%s. Expected volume id %s, "
+                                  "volume id %s found", brickinfo->hostname,
+                                  brickinfo->path,
+                                  uuid_utoa_r (volinfo->volume_id, volid),
+                                  uuid_utoa_r (volume_id, xattr_volid));
+                        ret = -1;
+                        goto out;
+                }
+#ifdef HAVE_BD_XLATOR
+                if (brickinfo->vg[0])
+                        caps = CAPS_BD | CAPS_THIN |
+                                CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+                /* Check for VG/thin pool if its BD volume */
+                if (brickinfo->vg[0]) {
+                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                        if (ret)
+                                goto out;
+                        /* if anyone of the brick does not have thin support,
+                           disable it for entire volume */
+                        caps &= brickinfo->caps;
+                } else
+                        caps = 0;
+#endif
+        }
+
+        volinfo->caps = caps;
+        ret = 0;
+out:
+        if (ret && (msg[0] != '\0')) {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        return ret;
+}
+
+int
+glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = -1;
+        char                                    *volname = NULL;
+        int                                     flags = 0;
+        gf_boolean_t                            exists = _gf_false;
+        gf_boolean_t                            is_run = _gf_false;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        char                                    msg[2048] = {0};
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
+        if (ret)
+                goto out;
+
+        exists = glusterd_check_volume_exists (volname);
+
+        if (!exists) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                ret = -1;
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        /* If 'force' flag is given, no check is required */
+        if (flags & GF_CLI_FLAG_OP_FORCE)
+                goto out;
+
+        if (_gf_false == glusterd_is_volume_started (volinfo)) {
+                snprintf (msg, sizeof(msg), "Volume %s "
+                          "is not in the started state", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                ret = -1;
+                goto out;
+        }
+        ret = glusterd_check_gsync_running (volinfo, &is_run);
+        if (ret && (is_run == _gf_false))
+                gf_log (this->name, GF_LOG_WARNING, "Unable to get the status"
+                        " of active "GEOREP" session");
+        if (is_run) {
+                gf_log (this->name, GF_LOG_WARNING, GEOREP" sessions active"
+                        "for the volume %s ", volname);
+                snprintf (msg, sizeof(msg), GEOREP" sessions are active "
+                          "for the volume '%s'.\nUse 'volume "GEOREP" "
+                          "status' command for more info. Use 'force' "
+                          "option to ignore and stop the volume.",
+                          volname);
+                ret = -1;
+                goto out;
+        }
+
+        if (glusterd_is_rb_ongoing (volinfo)) {
+                snprintf (msg, sizeof (msg), "Replace brick is in progress on "
+                          "volume %s. Please retry after replace-brick "
+                          "operation is committed or aborted", volname);
+                gf_log (this->name, GF_LOG_WARNING, "replace-brick in progress "
+                        "on volume %s", volname);
+                ret = -1;
+                goto out;
+        }
+
+        if (glusterd_is_defrag_on (volinfo)) {
+                snprintf (msg, sizeof(msg), "rebalance session is "
+                          "in progress for the volume '%s'", volname);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
+                ret = -1;
+                goto out;
+        }
+        if (volinfo->rep_brick.rb_status != GF_RB_STATUS_NONE) {
+                snprintf (msg, sizeof(msg), "replace-brick session is "
+                          "in progress for the volume '%s'", volname);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        if (msg[0] != 0)
+                *op_errstr = gf_strdup (msg);
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = 0;
+        char                                    *volname = NULL;
+        gf_boolean_t                            exists = _gf_false;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        char                                    msg[2048] = {0};
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        exists = glusterd_check_volume_exists (volname);
+        if (!exists) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                ret = -1;
+                goto out;
+        } else {
+                ret = 0;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_volume_started (volinfo)) {
+                snprintf (msg, sizeof (msg), "Volume %s has been started."
+                          "Volume needs to be stopped before deletion.",
+                          volname);
+                ret = -1;
+                goto out;
+        }
+
+        if (volinfo->snap_count > 0 || !list_empty(&volinfo->snap_volumes)) {
+                snprintf (msg, sizeof (msg), "Cannot delete Volume %s ,"
+                        "as it has %ld snapshots. "
+                        "To delete the volume, "
+                        "first delete all the snapshots under it.",
+                          volname, volinfo->snap_count);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        if (msg[0] != '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = 0;
+        char                                    *volname = NULL;
+        gf_boolean_t                            enabled = _gf_false;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        char                                    msg[2048];
+        glusterd_conf_t                         *priv = NULL;
+        dict_t                                  *opt_dict = NULL;
+        gf_xl_afr_op_t                          heal_op = GF_AFR_OP_INVALID;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        priv = this->private;
+        if (!priv) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "priv is NULL");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                ret = -1;
+                snprintf (msg, sizeof (msg), "Volume %s does not exist", volname);
+                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        if (!glusterd_is_volume_replicate (volinfo)) {
+                ret = -1;
+                snprintf (msg, sizeof (msg), "Volume %s is not of type "
+                          "replicate", volname);
+                *op_errstr = gf_strdup (msg);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
+                goto out;
+        }
+
+        if (!glusterd_is_volume_started (volinfo)) {
+                ret = -1;
+                snprintf (msg, sizeof (msg), "Volume %s is not started.",
+                          volname);
+                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        opt_dict = volinfo->dict;
+        if (!opt_dict) {
+                ret = 0;
+                goto out;
+        }
+
+        enabled = dict_get_str_boolean (opt_dict, "cluster.self-heal-daemon",
+                                        1);
+        if (!enabled) {
+                ret = -1;
+                snprintf (msg, sizeof (msg), "Self-heal-daemon is "
+                          "disabled. Heal will not be triggered on volume %s",
+                          volname);
+                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
+        if (ret || (heal_op == GF_AFR_OP_INVALID)) {
+                ret = -1;
+                *op_errstr = gf_strdup("Invalid heal-op");
+                gf_log (this->name, GF_LOG_WARNING, "%s", "Invalid heal-op");
+                goto out;
+        }
+
+        switch (heal_op) {
+                case GF_AFR_OP_INDEX_SUMMARY:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+                case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+                        break;
+                default:
+                        if (!glusterd_is_nodesvc_online("glustershd")){
+                                ret = -1;
+                                *op_errstr = gf_strdup ("Self-heal daemon is "
+                                                "not running. Check self-heal "
+                                                "daemon log file.");
+                                gf_log (this->name, GF_LOG_WARNING, "%s",
+                                        "Self-heal daemon is not running."
+                                        "Check self-heal daemon log file.");
+                                goto out;
+                        }
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+int
+glusterd_op_stage_statedump_volume (dict_t *dict, char **op_errstr)
+{
+        int                     ret = -1;
+        char                    *volname = NULL;
+        char                    *options = NULL;
+        int                     option_cnt = 0;
+        gf_boolean_t            is_running = _gf_false;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    msg[2408] = {0,};
+
+        ret = glusterd_op_statedump_volume_args_get (dict, &volname, &options,
+                                                     &option_cnt);
+        if (ret)
+                goto out;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Volume %s does not exist",
+                          volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        is_running = glusterd_is_volume_started (volinfo);
+        if (!is_running) {
+                snprintf (msg, sizeof(msg), "Volume %s is not in a started"
+                          " state", volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_stage_clearlocks_volume (dict_t *dict, char **op_errstr)
+{
+        int                     ret = -1;
+        char                    *volname = NULL;
+        char                    *path    = NULL;
+        char                    *type    = NULL;
+        char                    *kind    = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    msg[2048] = {0,};
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Failed to get volume name");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "path", &path);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Failed to get path");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "kind", &kind);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Failed to get kind");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "type", &type);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Failed to get type");
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof(msg), "Volume %s does not exist",
+                          volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = glusterd_validate_volume_id (dict, volinfo);
+        if (ret)
+                goto out;
+
+        if (!glusterd_is_volume_started (volinfo)) {
+                snprintf (msg, sizeof(msg), "Volume %s is not started",
+                          volname);
+                gf_log ("", GF_LOG_ERROR, "%s", msg);
+                *op_errstr = gf_strdup (msg);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_create_volume (dict_t *dict, char **op_errstr)
+{
+        int                   ret        = 0;
+        char                 *volname    = NULL;
+        glusterd_conf_t      *priv       = NULL;
+        glusterd_volinfo_t   *volinfo    = NULL;
+        gf_boolean_t          vol_added = _gf_false;
+        glusterd_brickinfo_t *brickinfo  = NULL;
+        xlator_t             *this       = NULL;
+        char                 *brick      = NULL;
+        int32_t               count      = 0;
+        int32_t               i          = 1;
+        char                 *bricks     = NULL;
+        char                 *brick_list = NULL;
+        char                 *free_ptr   = NULL;
+        char                 *saveptr    = NULL;
+        char                 *trans_type = NULL;
+        char                 *str        = NULL;
+        char                 *username   = NULL;
+        char                 *password   = NULL;
+        int                   caps       = 0;
+        char                  msg[1024] __attribute__((unused)) = {0, };
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = glusterd_volinfo_new (&volinfo);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to allocate memory for volinfo");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volname", &volname);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        strncpy (volinfo->volname, volname, GLUSTERD_MAX_VOLUME_NAME);
+        GF_ASSERT (volinfo->volname);
+
+        ret = dict_get_int32 (dict, "type", &volinfo->type);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get type of volume"
+                        " %s", volname);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "count", &volinfo->brick_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count of"
+                        " volume %s", volname);
+                goto out;
+        }
+
+        ret = dict_get_int32 (dict, "port", &volinfo->port);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get port");
+                goto out;
+        }
+
+        count = volinfo->brick_count;
+
+        ret = dict_get_str (dict, "bricks", &bricks);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks for "
+                        "volume %s", volname);
+                goto out;
+        }
+
+        /* replica-count 1 means, no replication, file is in one brick only */
+        volinfo->replica_count = 1;
+        /* stripe-count 1 means, no striping, file is present as a whole */
+        volinfo->stripe_count = 1;
+
+        if (GF_CLUSTER_TYPE_REPLICATE == volinfo->type) {
+                ret = dict_get_int32 (dict, "replica-count",
+                                      &volinfo->replica_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                 "replica count for volume %s", volname);
+                        goto out;
+                }
+        } else if (GF_CLUSTER_TYPE_STRIPE == volinfo->type) {
+                ret = dict_get_int32 (dict, "stripe-count",
+                                      &volinfo->stripe_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get stripe"
+                                " count for volume %s", volname);
+                        goto out;
+                }
+        } else if (GF_CLUSTER_TYPE_STRIPE_REPLICATE == volinfo->type) {
+                ret = dict_get_int32 (dict, "stripe-count",
+                                      &volinfo->stripe_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get stripe"
+                                " count for volume %s", volname);
+                        goto out;
+                }
+                ret = dict_get_int32 (dict, "replica-count",
+                                      &volinfo->replica_count);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "replica count for volume %s", volname);
+                        goto out;
+                }
+        }
+
+        /* dist-leaf-count is the count of brick nodes for a given
+           subvolume of distribute */
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+
+        /* subvol_count is the count of number of subvolumes present
+           for a given distribute volume */
+        volinfo->subvol_count = (volinfo->brick_count /
+                                 volinfo->dist_leaf_count);
+
+        /* Keep sub-count same as earlier, for the sake of backward
+           compatibility */
+        if (volinfo->dist_leaf_count > 1)
+                volinfo->sub_count = volinfo->dist_leaf_count;
+
+        ret = dict_get_str (dict, "transport", &trans_type);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to get transport type of volume %s", volname);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "volume-id", &str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to get volume-id of volume %s", volname);
+                goto out;
+        }
+        ret = uuid_parse (str, volinfo->volume_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to parse uuid %s of volume %s", str, volname);
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "internal-username", &username);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to get internal username of volume %s",
+                        volname);
+                goto out;
+        }
+        glusterd_auth_set_username (volinfo, username);
+
+        ret = dict_get_str (dict, "internal-password", &password);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to get internal password of volume %s",
+                        volname);
+                goto out;
+        }
+        glusterd_auth_set_password (volinfo, password);
+
+        if (strcasecmp (trans_type, "rdma") == 0) {
+                volinfo->transport_type = GF_TRANSPORT_RDMA;
+                volinfo->nfs_transport_type = GF_TRANSPORT_RDMA;
+        } else if (strcasecmp (trans_type, "tcp") == 0) {
+                volinfo->transport_type = GF_TRANSPORT_TCP;
+                volinfo->nfs_transport_type = GF_TRANSPORT_TCP;
+        } else {
+                volinfo->transport_type = GF_TRANSPORT_BOTH_TCP_RDMA;
+                volinfo->nfs_transport_type = GF_DEFAULT_NFS_TRANSPORT;
+        }
+
+        if (bricks) {
+                brick_list = gf_strdup (bricks);
+                free_ptr = brick_list;
+        }
+
+        if (count)
+                brick = strtok_r (brick_list+1, " \n", &saveptr);
+        caps = CAPS_BD | CAPS_THIN | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
+
+        while ( i <= count) {
+                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_resolve_brick (brickinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
+                                brickinfo->hostname, brickinfo->path);
+                        goto out;
+                }
+
+#ifdef HAVE_BD_XLATOR
+                if (!uuid_compare (brickinfo->uuid, MY_UUID)) {
+                        if (brickinfo->vg[0]) {
+                                ret = glusterd_is_valid_vg (brickinfo, 0, msg);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR, "%s",
+                                                msg);
+                                        goto out;
+                                }
+
+                                /* if anyone of the brick does not have thin
+                                   support, disable it for entire volume */
+                                caps &= brickinfo->caps;
+
+
+                        } else
+                                caps = 0;
+                }
+#endif
+
+                list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
+                brick = strtok_r (NULL, " \n", &saveptr);
+                i++;
+        }
+
+        gd_update_volume_op_versions (volinfo);
+
+        volinfo->caps = caps;
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                glusterd_store_delete_volume (volinfo);
+                *op_errstr = gf_strdup ("Failed to store the Volume information");
+                goto out;
+        }
+
+        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        if (ret) {
+                *op_errstr = gf_strdup ("Failed to create volume files");
+                goto out;
+        }
+
+        volinfo->rebal.defrag_status = 0;
+        list_add_tail (&volinfo->vol_list, &priv->volumes);
+        vol_added = _gf_true;
+
+out:
+        GF_FREE(free_ptr);
+        if (!vol_added && volinfo)
+                glusterd_volinfo_delete (volinfo);
+        return ret;
+}
+
+int
+glusterd_op_start_volume (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = 0;
+        char                                    *volname = NULL;
+        int                                     flags = 0;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        glusterd_brickinfo_t                    *brickinfo = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = glusterd_op_start_volume_args_get (dict, &volname, &flags);
+        if (ret)
+                goto out;
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_brick_start (volinfo, brickinfo, _gf_true);
+                /* If 'force' try to start all bricks regardless of success or
+                 * failure
+                 */
+                if (!(flags & GF_CLI_FLAG_OP_FORCE) && ret)
+                        goto out;
+        }
+
+        glusterd_set_volume_status (volinfo, GLUSTERD_STATUS_STARTED);
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+                goto out;
+
+        ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
+        return ret;
+}
+
+int
+glusterd_stop_volume (glusterd_volinfo_t *volinfo)
+{
+        int                     ret             = -1;
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        xlator_t                *this           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                ret = glusterd_brick_stop (volinfo, brickinfo, _gf_false);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
+                                "brick (%s)", brickinfo->path);
+                        goto out;
+                }
+        }
+
+        glusterd_set_volume_status (volinfo, GLUSTERD_STATUS_STOPPED);
+
+        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo of "
+                        "%s volume", volinfo->volname);
+                goto out;
+        }
+
+        ret = glusterd_nodesvcs_handle_graph_change (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to notify graph "
+                        "change for %s volume", volinfo->volname);
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+
+int
+glusterd_op_stop_volume (dict_t *dict)
+{
+        int                                     ret = 0;
+        int                                     flags = 0;
+        char                                    *volname = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
+        if (ret)
+                goto out;
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        ret = glusterd_stop_volume (volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to stop %s volume",
+                        volname);
+                goto out;
+        }
+out:
+        return ret;
+}
+
+int
+glusterd_op_delete_volume (dict_t *dict)
+{
+        int                                     ret = 0;
+        char                                    *volname = NULL;
+        glusterd_conf_t                         *priv = NULL;
+        glusterd_volinfo_t                      *volinfo = NULL;
+        xlator_t                                *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret  = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
+                        volname);
+                goto out;
+        }
+
+        ret = glusterd_delete_volume (volinfo);
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_op_heal_volume (dict_t *dict, char **op_errstr)
+{
+        int                                     ret = 0;
+        /* Necessary subtasks of heal are completed in brick op */
+
+        return ret;
+}
+
+int
+glusterd_op_statedump_volume (dict_t *dict, char **op_errstr)
+{
+        int                     ret = 0;
+        char                    *volname = NULL;
+        char                    *options = NULL;
+        int                     option_cnt = 0;
+        glusterd_volinfo_t      *volinfo = NULL;
+        glusterd_brickinfo_t    *brickinfo = NULL;
+
+        ret = glusterd_op_statedump_volume_args_get (dict, &volname, &options,
+                                                     &option_cnt);
+        if (ret)
+                goto out;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret)
+                goto out;
+        gf_log ("", GF_LOG_DEBUG, "Performing statedump on volume %s", volname);
+        if (strstr (options, "nfs") != NULL) {
+                ret = glusterd_nfs_statedump (options, option_cnt, op_errstr);
+                if (ret)
+                        goto out;
+        } else {
+                list_for_each_entry (brickinfo, &volinfo->bricks,
+                                                        brick_list) {
+                        ret = glusterd_brick_statedump (volinfo, brickinfo,
+                                                        options, option_cnt,
+                                                        op_errstr);
+                        /* Let us take the statedump of other bricks instead of
+                         * exiting, if statedump of this brick fails.
+                         */
+                        if (ret)
+                                gf_log (THIS->name, GF_LOG_WARNING, "could not "
+                                        "take the statedump of the brick %s:%s."
+                                        " Proceeding to other bricks",
+                                        brickinfo->hostname, brickinfo->path);
+                }
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_clearlocks_send_cmd (glusterd_volinfo_t *volinfo, char *cmd,
+                              char *path, char *result, char *errstr,
+                              int err_len, char *mntpt)
+{
+        int               ret                   = -1;
+        glusterd_conf_t  *priv                  = NULL;
+        char             abspath[PATH_MAX]      = {0, };
+
+        priv = THIS->private;
+
+        snprintf (abspath, sizeof (abspath), "%s/%s", mntpt, path);
+        ret = sys_lgetxattr (abspath, cmd, result, PATH_MAX);
+        if (ret < 0) {
+                snprintf (errstr, err_len, "clear-locks getxattr command "
+                          "failed. Reason: %s", strerror (errno));
+                gf_log (THIS->name, GF_LOG_DEBUG, "%s", errstr);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_clearlocks_rmdir_mount (glusterd_volinfo_t *volinfo, char *mntpt)
+{
+        int              ret               = -1;
+        glusterd_conf_t *priv              = NULL;
+
+        priv = THIS->private;
+
+        ret = rmdir (mntpt);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "rmdir failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+glusterd_clearlocks_unmount (glusterd_volinfo_t *volinfo, char *mntpt)
+{
+        glusterd_conf_t  *priv               = NULL;
+        runner_t          runner             = {0,};
+        int               ret                = 0;
+
+        priv = THIS->private;
+
+        /*umount failures are ignored. Using stat we could have avoided
+         * attempting to unmount a non-existent filesystem. But a failure of
+         * stat() on mount can be due to network failures.*/
+
+        runinit (&runner);
+        runner_add_args (&runner, "/bin/umount", "-f", NULL);
+        runner_argprintf (&runner, "%s", mntpt);
+
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                ret = 0;
+                gf_log ("", GF_LOG_DEBUG,
+                        "umount failed on maintenance client");
+        }
+
+        return;
+}
+
+int
+glusterd_clearlocks_create_mount (glusterd_volinfo_t *volinfo, char **mntpt)
+{
+        int              ret                    = -1;
+        glusterd_conf_t *priv                   = NULL;
+        char             template[PATH_MAX]     = {0,};
+        char            *tmpl                   = NULL;
+
+        priv = THIS->private;
+
+        snprintf (template, sizeof (template), "/tmp/%s.XXXXXX",
+                  volinfo->volname);
+        tmpl = mkdtemp (template);
+        if (!tmpl) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "Couldn't create temporary "
+                        "mount directory. Reason %s", strerror (errno));
+                goto out;
+        }
+
+        *mntpt = gf_strdup (tmpl);
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_clearlocks_mount (glusterd_volinfo_t *volinfo, char **xl_opts,
+                           char *mntpt)
+{
+        int             ret                             = -1;
+        int             i                               = 0;
+        glusterd_conf_t *priv                           = NULL;
+        runner_t        runner                          = {0,};
+        char            client_volfpath[PATH_MAX]       = {0,};
+        char            self_heal_opts[3][1024]      = {"*replicate*.data-self-heal=off",
+                                                        "*replicate*.metadata-self-heal=off",
+                                                        "*replicate*.entry-self-heal=off"};
+
+        priv = THIS->private;
+
+        runinit (&runner);
+        glusterd_get_trusted_client_filepath (client_volfpath, volinfo,
+                                      volinfo->transport_type);
+        runner_add_args (&runner, SBIN_DIR"/glusterfs", "-f", NULL);
+        runner_argprintf (&runner, "%s", client_volfpath);
+        runner_add_arg (&runner, "-l");
+        runner_argprintf (&runner, DEFAULT_LOG_FILE_DIRECTORY
+                          "/%s-clearlocks-mnt.log", volinfo->volname);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
+        for (i = 0; i < volinfo->brick_count && xl_opts[i]; i++) {
+                runner_add_arg (&runner, "--xlator-option");
+                runner_argprintf (&runner, "%s", xl_opts[i]);
+        }
+
+        for (i = 0; i < 3; i++) {
+                runner_add_args (&runner, "--xlator-option",
+                                 self_heal_opts[i], NULL);
+        }
+
+        runner_argprintf (&runner, "%s", mntpt);
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+        synclock_lock (&priv->big_lock);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_DEBUG,
+                        "Could not start glusterfs");
+                goto out;
+        }
+        gf_log (THIS->name, GF_LOG_DEBUG,
+                "Started glusterfs successfully");
+
+out:
+        return ret;
+}
+
+int
+glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
+                                            char **xl_opts)
+{
+        glusterd_brickinfo_t    *brickinfo      = NULL;
+        glusterd_conf_t         *priv           = NULL;
+        int                     index           = 0;
+        int                     ret             = -1;
+        int                     i               = 0;
+        int                     port            = 0;
+
+        GF_ASSERT (xl_opts);
+        if (!xl_opts) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "Should pass non-NULL "
+                        "xl_opts");
+                goto out;
+        }
+
+        priv = THIS->private;
+
+        index = -1;
+        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
+                index++;
+                if (uuid_compare (brickinfo->uuid, MY_UUID))
+                        continue;
+
+                port = pmap_registry_search (THIS, brickinfo->path,
+                                             GF_PMAP_PORT_BRICKSERVER);
+                if (!port) {
+                        ret = -1;
+                        gf_log (THIS->name, GF_LOG_DEBUG, "Couldn't get port "
+                                " for brick %s:%s", brickinfo->hostname,
+                                brickinfo->path);
+                        goto out;
+                }
+
+                ret = gf_asprintf (&xl_opts[i], "%s-client-%d.remote-port=%d",
+                                   volinfo->volname, index, port);
+                if (ret == -1) {
+                        xl_opts[i] = NULL;
+                        goto out;
+                }
+                i++;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+{
+        int32_t                         ret                 = -1;
+        int                             i                   = 0;
+        char                            *volname            = NULL;
+        char                            *path               = NULL;
+        char                            *kind               = NULL;
+        char                            *type               = NULL;
+        char                            *opts               = NULL;
+        char                            *cmd_str            = NULL;
+        char                            *free_ptr           = NULL;
+        char                            msg[PATH_MAX]       = {0,};
+        char                            result[PATH_MAX]    = {0,};
+        char                            *mntpt              = NULL;
+        char                            **xl_opts           = NULL;
+        glusterd_volinfo_t              *volinfo            = NULL;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get volume name");
+                goto out;
+        }
+        gf_log ("", GF_LOG_DEBUG, "Performing clearlocks on volume %s", volname);
+
+        ret = dict_get_str (dict, "path", &path);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get path");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "kind", &kind);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get kind");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "type", &type);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get type");
+                goto out;
+        }
+
+        ret = dict_get_str (dict, "opts", &opts);
+        if (ret)
+                ret = 0;
+
+        gf_log (THIS->name, GF_LOG_INFO, "Received clear-locks request for "
+                "volume %s with kind %s type %s and options %s", volname,
+                kind, type, opts);
+
+        if (opts)
+                ret = gf_asprintf (&cmd_str, GF_XATTR_CLRLK_CMD".t%s.k%s.%s",
+                                   type, kind, opts);
+        else
+                ret = gf_asprintf (&cmd_str, GF_XATTR_CLRLK_CMD".t%s.k%s",
+                                   type, kind);
+        if (ret == -1)
+                goto out;
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Volume %s doesn't exist.",
+                          volname);
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        xl_opts = GF_CALLOC (volinfo->brick_count+1, sizeof (char*),
+                             gf_gld_mt_charptr);
+        if (!xl_opts)
+                goto out;
+
+        ret = glusterd_clearlocks_get_local_client_ports (volinfo, xl_opts);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Couldn't get port numbers of "
+                          "local bricks");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_clearlocks_create_mount (volinfo, &mntpt);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Creating mount directory "
+                          "for clear-locks failed.");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_clearlocks_mount (volinfo, xl_opts, mntpt);
+        if (ret) {
+                snprintf (msg, sizeof (msg), "Failed to mount clear-locks "
+                          "maintenance client.");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto out;
+        }
+
+        ret = glusterd_clearlocks_send_cmd (volinfo, cmd_str, path, result,
+                                            msg, sizeof (msg), mntpt);
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+                goto umount;
+        }
+
+        free_ptr = gf_strdup(result);
+        if (dict_set_dynstr (rsp_dict, "lk-summary", free_ptr)) {
+                GF_FREE (free_ptr);
+                snprintf (msg, sizeof (msg), "Failed to set clear-locks "
+                          "result");
+                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
+        }
+
+umount:
+        glusterd_clearlocks_unmount (volinfo, mntpt);
+
+        if (glusterd_clearlocks_rmdir_mount (volinfo, mntpt))
+                gf_log (THIS->name, GF_LOG_WARNING, "Couldn't unmount "
+                        "clear-locks mount point");
+
+out:
+        if (ret)
+                *op_errstr = gf_strdup (msg);
+
+        if (xl_opts) {
+                for (i = 0; i < volinfo->brick_count && xl_opts[i]; i++)
+                        GF_FREE (xl_opts[i]);
+                GF_FREE (xl_opts);
+        }
+
+        GF_FREE (cmd_str);
+
+        GF_FREE (mntpt);
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
new file mode 100644
index 000000000..665a8b298
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -0,0 +1,1452 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterd-volgen.h"
+#include "glusterd-utils.h"
+
+static int
+check_dict_key_value (dict_t *dict, char *key, char *value)
+{
+        glusterd_conf_t     *priv          = NULL;
+        int                  ret           = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (!dict) {
+                gf_log (this->name, GF_LOG_ERROR, "Received Empty Dict.");
+                ret = -1;
+                goto out;
+        }
+
+        if (!key) {
+                gf_log (this->name, GF_LOG_ERROR, "Received Empty Key.");
+                ret = -1;
+                goto out;
+        }
+
+        if (!value) {
+                gf_log (this->name, GF_LOG_ERROR, "Received Empty Value.");
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+get_volname_volinfo (dict_t *dict, char **volname, glusterd_volinfo_t **volinfo)
+{
+        glusterd_conf_t     *priv          = NULL;
+        int                  ret           = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = dict_get_str (dict, "volname", volname);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (*volname, volinfo);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_cache_max_min_size (dict_t *dict, char *key, char *value,
+                             char **op_errstr)
+{
+        char                *current_max_value = NULL;
+        char                *current_min_value = NULL;
+        char                 errstr[2048]  = "";
+        char                *volname       = NULL;
+        glusterd_conf_t     *priv          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        uint64_t             max_value     = 0;
+        uint64_t             min_value     = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        if ((!strcmp (key, "performance.cache-min-file-size")) ||
+            (!strcmp (key, "cache-min-file-size"))) {
+                glusterd_volinfo_get (volinfo,
+                                      "performance.cache-max-file-size",
+                                      &current_max_value);
+                if (current_max_value) {
+                        gf_string2bytesize (current_max_value, &max_value);
+                        gf_string2bytesize (value, &min_value);
+                        current_min_value = value;
+                }
+        } else  if ((!strcmp (key, "performance.cache-max-file-size")) ||
+                    (!strcmp (key, "cache-max-file-size"))) {
+                glusterd_volinfo_get (volinfo,
+                                      "performance.cache-min-file-size",
+                                      &current_min_value);
+                if (current_min_value) {
+                        gf_string2bytesize (current_min_value, &min_value);
+                        gf_string2bytesize (value, &max_value);
+                        current_max_value = value;
+                }
+        }
+
+        if (min_value > max_value) {
+                snprintf (errstr, sizeof (errstr),
+                          "cache-min-file-size (%s) is greater than "
+                          "cache-max-file-size (%s)",
+                          current_min_value, current_max_value);
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_quota (dict_t *dict, char *key, char *value,
+                char **op_errstr)
+{
+        char                 errstr[2048] = "";
+        char                *volname      = NULL;
+        glusterd_conf_t     *priv         = NULL;
+        glusterd_volinfo_t  *volinfo      = NULL;
+        int                  ret          = 0;
+        xlator_t            *this         = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get the quota status");
+                goto out;
+        }
+
+        if (ret == _gf_false) {
+                snprintf (errstr, sizeof (errstr),
+                          "Cannot set %s. Enable quota first.", key);
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_stripe (dict_t *dict, char *key, char *value, char **op_errstr)
+{
+        char                 errstr[2048]  = "";
+        char                *volname       = NULL;
+        glusterd_conf_t     *priv          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        if (volinfo->stripe_count == 1) {
+                snprintf (errstr, sizeof (errstr),
+                          "Cannot set %s for a non-stripe volume.", key);
+                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+               goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+static int
+validate_subvols_per_directory (dict_t *dict, char *key, char *value,
+                                char **op_errstr)
+{
+        char                 errstr[2048]  = "";
+        char                *volname       = NULL;
+        glusterd_conf_t     *priv          = NULL;
+        glusterd_volinfo_t  *volinfo       = NULL;
+        int                  ret           = 0;
+        int                  subvols       = 0;
+        xlator_t            *this          = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        ret = check_dict_key_value (dict, key, value);
+        if (ret)
+                goto out;
+
+        ret = get_volname_volinfo (dict, &volname, &volinfo);
+        if (ret)
+                goto out;
+
+        subvols = atoi(value);
+
+        /* Checking if the subvols-per-directory exceed the total
+           number of subvolumes. */
+        if (subvols > volinfo->subvol_count) {
+                snprintf (errstr, sizeof(errstr),
+                          "subvols-per-directory(%d) is greater "
+                          "than the number of subvolumes(%d).",
+                          subvols, volinfo->subvol_count);
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s.", errstr);
+                *op_errstr = gf_strdup (errstr);
+                ret = -1;
+                goto out;
+        }
+
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+
+        return ret;
+}
+
+
+/* dispatch table for VOLUME SET
+ * -----------------------------
+ *
+ * Format of entries:
+ *
+ * First field is the <key>, for the purpose of looking it up
+ * in volume dictionary. Each <key> is of the format "<domain>.<specifier>".
+ *
+ * Second field is <voltype>.
+ *
+ * Third field is <option>, if its unset, it's assumed to be
+ * the same as <specifier>.
+ *
+ * Fourth field is <value>. In this context they are used to specify
+ * a default. That is, even the volume dict doesn't have a value,
+ * we procced as if the default value were set for it.
+ *
+ * Fifth field is <doctype>, which decides if the option is public and available
+ * in "set help" or not. "NO_DOC" entries are not part of the public interface
+ * and are subject to change at any time. This also decides if an option is
+ * global (apllies to all volumes) or normal (applies to only specified volume).
+ *
+ * Sixth field is <flags>.
+ *
+ * Seventh field is <op-version>.
+ *
+ * Eight field is description of option: If NULL, tried to fetch from
+ * translator code's xlator_options table.
+ *
+ * Nineth field is validation function: If NULL, xlator's option specific
+ * validation will be tried, otherwise tried at glusterd code itself.
+ *
+ * There are two type of entries: basic and special.
+ *
+ * - Basic entries are the ones where the <option> does _not_ start with
+ *   the bang! character ('!').
+ *
+ *   In their case, <option> is understood as an option for an xlator of
+ *   type <voltype>. Their effect is to copy over the volinfo->dict[<key>]
+ *   value to all graph nodes of type <voltype> (if such a value is set).
+ *
+ *   You are free to add entries of this type, they will become functional
+ *   just by being present in the table.
+ *
+ * - Special entries where the <option> starts with the bang!.
+ *
+ *   They are not applied to all graphs during generation, and you cannot
+ *   extend them in a trivial way which could be just picked up. Better
+ *   not touch them unless you know what you do.
+ *
+ *
+ * Another kind of grouping for options, according to visibility:
+ *
+ * - Exported: one which is used in the code. These are characterized by
+ *   being used a macro as <key> (of the format VKEY_..., defined in
+ *   glusterd-volgen.h
+ *
+ * - Non-exported: the rest; these have string literal <keys>.
+ *
+ * Adhering to this policy, option name changes shall be one-liners.
+ *
+ */
+
+struct volopt_map_entry glusterd_volopt_map[] = {
+        /* DHT xlator options */
+        { .key        = "cluster.lookup-unhashed",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.min-free-disk",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.min-free-inodes",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.rebalance-stats",
+          .voltype    = "cluster/distribute",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "cluster.subvols-per-directory",
+          .voltype     = "cluster/distribute",
+          .option      = "directory-layout-spread",
+          .op_version  = 2,
+          .validate_fn = validate_subvols_per_directory,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.readdir-optimize",
+          .voltype    = "cluster/distribute",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.rsync-hash-regex",
+          .voltype    = "cluster/distribute",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.extra-hash-regex",
+          .voltype    = "cluster/distribute",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.dht-xattr-name",
+          .voltype    = "cluster/distribute",
+          .option     = "xattr-name",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* NUFA xlator options (Distribute special case) */
+        { .key        = "cluster.nufa",
+          .voltype    = "cluster/distribute",
+          .option     = "!nufa",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.local-volume-name",
+          .voltype    = "cluster/nufa",
+          .option     = "local-volume-name",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Switch xlator options (Distribute special case) */
+        { .key        = "cluster.switch",
+          .voltype    = "cluster/distribute",
+          .option     = "!switch",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.switch-pattern",
+          .voltype    = "cluster/switch",
+          .option     = "pattern.switch.case",
+          .type       = NO_DOC,
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* AFR xlator options */
+        { .key        = "cluster.entry-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-subvolume",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags     = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-subvolume-index",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.read-hash-mode",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.background-self-heal-count",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.metadata-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.entry-self-heal",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key           = "cluster.self-heal-daemon",
+          .voltype       = "cluster/replicate",
+          .option        = "!self-heal-daemon",
+          .op_version    = 1
+        },
+        { .key        = "cluster.heal-timeout",
+          .voltype    = "cluster/replicate",
+          .option     = "!heal-timeout",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.strict-readdir",
+          .voltype    = "cluster/replicate",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.self-heal-window-size",
+          .voltype    = "cluster/replicate",
+          .option     = "data-self-heal-window-size",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.metadata-change-log",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.data-self-heal-algorithm",
+          .voltype    = "cluster/replicate",
+          .option     = "data-self-heal-algorithm",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.eager-lock",
+          .voltype    = "cluster/replicate",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.quorum-type",
+          .voltype    = "cluster/replicate",
+          .option     = "quorum-type",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.quorum-count",
+          .voltype    = "cluster/replicate",
+          .option     = "quorum-count",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.choose-local",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.self-heal-readdir-size",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.post-op-delay-secs",
+          .voltype    = "cluster/replicate",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.readdir-failover",
+          .voltype    = "cluster/replicate",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.ensure-durability",
+          .voltype    = "cluster/replicate",
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Stripe xlator options */
+        { .key         = "cluster.stripe-block-size",
+          .voltype     = "cluster/stripe",
+          .option      = "block-size",
+          .op_version  = 1,
+          .validate_fn = validate_stripe,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "cluster.stripe-coalesce",
+          .voltype    = "cluster/stripe",
+          .option     = "coalesce",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* IO-stats xlator options */
+        { .key         = VKEY_DIAG_LAT_MEASUREMENT,
+          .voltype     = "debug/io-stats",
+          .option      = "latency-measurement",
+          .value       = "off",
+          .op_version  = 1
+        },
+        { .key         = "diagnostics.dump-fd-stats",
+          .voltype     = "debug/io-stats",
+          .op_version  = 1
+        },
+        { .key         = VKEY_DIAG_CNT_FOP_HITS,
+          .voltype     = "debug/io-stats",
+          .option      = "count-fop-hits",
+          .value       = "off",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "diagnostics.brick-log-level",
+          .voltype     = "debug/io-stats",
+          .option      = "!brick-log-level",
+          .op_version  = 1
+        },
+        { .key        = "diagnostics.client-log-level",
+          .voltype    = "debug/io-stats",
+          .option     = "!client-log-level",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "diagnostics.brick-sys-log-level",
+          .voltype     = "debug/io-stats",
+          .option      = "!sys-log-level",
+          .op_version  = 1
+        },
+        { .key        = "diagnostics.client-sys-log-level",
+          .voltype    = "debug/io-stats",
+          .option     = "!sys-log-level",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* IO-cache xlator options */
+        { .key         = "performance.cache-max-file-size",
+          .voltype     = "performance/io-cache",
+          .option      = "max-file-size",
+          .op_version  = 1,
+          .validate_fn = validate_cache_max_min_size,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "performance.cache-min-file-size",
+          .voltype     = "performance/io-cache",
+          .option      = "min-file-size",
+          .op_version  = 1,
+          .validate_fn = validate_cache_max_min_size,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-refresh-timeout",
+          .voltype    = "performance/io-cache",
+          .option     = "cache-timeout",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-priority",
+          .voltype    = "performance/io-cache",
+          .option     = "priority",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.cache-size",
+          .voltype    = "performance/io-cache",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* IO-threads xlator options */
+        { .key         = "performance.io-thread-count",
+          .voltype     = "performance/io-threads",
+          .option      = "thread-count",
+          .op_version  = 1
+        },
+        { .key         = "performance.high-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.normal-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.low-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.least-prio-threads",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.enable-least-priority",
+          .voltype     = "performance/io-threads",
+          .op_version  = 1
+        },
+        { .key         = "performance.least-rate-limit",
+          .voltype     = "performance/io-threads",
+          .op_version  = 2
+        },
+
+        /* Other perf xlators' options */
+        { .key        = "performance.cache-size",
+          .voltype    = "performance/quick-read",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.flush-behind",
+          .voltype    = "performance/write-behind",
+          .option     = "flush-behind",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.write-behind-window-size",
+          .voltype    = "performance/write-behind",
+          .option     = "cache-size",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.strict-o-direct",
+          .voltype    = "performance/write-behind",
+          .option     = "strict-O_DIRECT",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.strict-write-ordering",
+          .voltype    = "performance/write-behind",
+          .option     = "strict-write-ordering",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.lazy-open",
+          .voltype    = "performance/open-behind",
+          .option     = "lazy-open",
+          .op_version = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.read-ahead-page-count",
+          .voltype    = "performance/read-ahead",
+          .option     = "page-count",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "performance.md-cache-timeout",
+          .voltype    = "performance/md-cache",
+          .option     = "md-cache-timeout",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+ 	/* Crypt xlator options */
+
+	{ .key         = "features.encryption",
+	  .voltype     = "encryption/crypt",
+	  .option      = "!feat",
+	  .value       = "off",
+	  .op_version  = 3,
+	  .description = "enable/disable client-side encryption for "
+                         "the volume.",
+	  .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+	},
+
+        { .key         = "encryption.master-key",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "encryption.data-key-size",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "encryption.block-size",
+          .voltype     = "encryption/crypt",
+          .op_version  = 3,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Client xlator options */
+        { .key        = "network.frame-timeout",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.ping-timeout",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.tcp-window-size",
+          .voltype    = "protocol/client",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "features.lock-heal",
+          .voltype    = "protocol/client",
+          .option     = "lk-heal",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "features.grace-timeout",
+          .voltype    = "protocol/client",
+          .option     = "grace-timeout",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "client.ssl",
+          .voltype    = "protocol/client",
+          .option     = "transport.socket.ssl-enabled",
+          .type       = NO_DOC,
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key        = "network.remote-dio",
+          .voltype    = "protocol/client",
+          .option     = "filter-O_DIRECT",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Server xlator options */
+        { .key         = "network.tcp-window-size",
+          .voltype     = "protocol/server",
+          .op_version  = 1
+        },
+        { .key         = "network.inode-lru-limit",
+          .voltype     = "protocol/server",
+          .op_version  = 1
+        },
+        { .key         = AUTH_ALLOW_MAP_KEY,
+          .voltype     = "protocol/server",
+          .option      = "!server-auth",
+          .value       = "*",
+          .op_version  = 1
+        },
+        { .key         = AUTH_REJECT_MAP_KEY,
+          .voltype     = "protocol/server",
+          .option      = "!server-auth",
+          .op_version  = 1
+        },
+        { .key         = "transport.keepalive",
+          .voltype     = "protocol/server",
+          .option      = "transport.socket.keepalive",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "server.allow-insecure",
+          .voltype     = "protocol/server",
+          .option      = "rpc-auth-allow-insecure",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "server.root-squash",
+          .voltype     = "protocol/server",
+          .option      = "root-squash",
+          .op_version  = 2
+        },
+        { .key         = "server.statedump-path",
+          .voltype     = "protocol/server",
+          .option      = "statedump-path",
+          .op_version  = 1
+        },
+        { .key         = "server.outstanding-rpc-limit",
+          .voltype     = "protocol/server",
+          .option      = "rpc.outstanding-rpc-limit",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "features.lock-heal",
+          .voltype     = "protocol/server",
+          .option      = "lk-heal",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "features.grace-timeout",
+          .voltype     = "protocol/server",
+          .option      = "grace-timeout",
+          .type        = NO_DOC,
+          .op_version  = 1
+        },
+        { .key         = "server.ssl",
+          .voltype     = "protocol/server",
+          .option      = "transport.socket.ssl-enabled",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+
+        /* Performance xlators enable/disbable options */
+        { .key         = "performance.write-behind",
+          .voltype     = "performance/write-behind",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable write-behind translator in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "performance.read-ahead",
+          .voltype     = "performance/read-ahead",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable read-ahead translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "performance.readdir-ahead",
+          .voltype     = "performance/readdir-ahead",
+          .option      = "!perf",
+          .value       = "off",
+          .op_version  = 3,
+          .description = "enable/disable readdir-ahead translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+
+        { .key         = "performance.io-cache",
+          .voltype     = "performance/io-cache",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable io-cache translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "performance.quick-read",
+          .voltype     = "performance/quick-read",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable quick-read translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+
+        },
+        { .key         = "performance.open-behind",
+          .voltype     = "performance/open-behind",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 2,
+          .description = "enable/disable open-behind translator in the volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+
+        },
+        { .key         = "performance.stat-prefetch",
+          .voltype     = "performance/md-cache",
+          .option      = "!perf",
+          .value       = "on",
+          .op_version  = 1,
+          .description = "enable/disable meta-data caching translator in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "performance.client-io-threads",
+          .voltype     = "performance/io-threads",
+          .option      = "!perf",
+          .value       = "off",
+          .op_version  = 1,
+          .description = "enable/disable io-threads translator in the client "
+                         "graph of volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.write-behind",
+          .voltype    = "performance/write-behind",
+          .option     = "!nfsperf",
+          .value      = "on",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.read-ahead",
+          .voltype    = "performance/read-ahead",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.io-cache",
+          .voltype    = "performance/io-cache",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.quick-read",
+          .voltype    = "performance/quick-read",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.stat-prefetch",
+          .voltype    = "performance/md-cache",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.nfs.io-threads",
+          .voltype    = "performance/io-threads",
+          .option     = "!nfsperf",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "performance.force-readdirp",
+          .voltype    = "performance/md-cache",
+          .option     = "force-readdirp",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+
+	/* Feature translators */
+        { .key         = "features.file-snapshot",
+          .voltype     = "features/qemu-block",
+          .option      = "!feat",
+          .value       = "off",
+          .op_version  = 3,
+          .description = "enable/disable file-snapshot feature in the "
+                         "volume.",
+          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+
+#ifdef HAVE_LIB_Z
+        /* Compressor-decompressor xlator options
+         * defaults used from xlator/feature/compress/src/cdc.h
+         */
+        { .key         = "features.compress",
+          .voltype     = "features/cdc",
+          .option      = "!compress",
+          .value       = "off",
+          .type        = NO_DOC,
+          .op_version  = 2,
+          .description = "enable/disable compression translator"
+        },
+        { .key         = "compress.mode",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.window-size",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.mem-level",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.min-size",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.compression-level",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "compress.debug",
+          .voltype     = "features/cdc",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+ #endif
+
+        /* Quota xlator options */
+        { .key        = VKEY_FEATURES_LIMIT_USAGE,
+          .voltype    = "features/quota",
+          .option     = "limit-set",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "features.quota-timeout",
+          .voltype     = "features/quota",
+          .option      = "timeout",
+          .value       = "0",
+          .op_version  = 1,
+          .validate_fn = validate_quota,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "features.quota-deem-statfs",
+          .voltype     = "features/quota",
+          .option      = "deem-statfs",
+          .value       = "off",
+          .type        = DOC,
+          .op_version  = 3,
+          .validate_fn = validate_quota,
+          .flags       = OPT_FLAG_CLIENT_OPT
+        },
+
+        /* Marker xlator options */
+        { .key         = VKEY_MARKER_XTIME,
+          .voltype     = "features/marker",
+          .option      = "xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 1
+        },
+        { .key         = VKEY_MARKER_XTIME,
+          .voltype     = "features/marker",
+          .option      = "!xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 1
+        },
+        { .key         = VKEY_MARKER_XTIME_FORCE,
+          .voltype     = "features/marker",
+          .option      = "gsync-force-xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 2
+        },
+        { .key         = VKEY_MARKER_XTIME_FORCE,
+          .voltype     = "features/marker",
+          .option      = "!gsync-force-xtime",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 2
+        },
+        { .key         = VKEY_FEATURES_QUOTA,
+          .voltype     = "features/marker",
+          .option      = "quota",
+          .value       = "off",
+          .type        = NO_DOC,
+          .flags       = OPT_FLAG_FORCE,
+          .op_version  = 1
+        },
+
+        /* Debug xlators options */
+        { .key        = "debug.trace",
+          .voltype    = "debug/trace",
+          .option     = "!debug",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "debug.log-history",
+          .voltype     = "debug/trace",
+          .option      = "log-history",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "debug.log-file",
+          .voltype     = "debug/trace",
+          .option      = "log-file",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "debug.exclude-ops",
+          .voltype     = "debug/trace",
+          .option      = "exclude-ops",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "debug.include-ops",
+          .voltype     = "debug/trace",
+          .option      = "include-ops",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key        = "debug.error-gen",
+          .voltype    = "debug/error-gen",
+          .option     = "!debug",
+          .value      = "off",
+          .type       = NO_DOC,
+          .op_version = 1,
+          .flags      = OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "debug.error-failure",
+          .voltype     = "debug/error-gen",
+          .option      = "failure",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "debug.error-number",
+          .voltype     = "debug/error-gen",
+          .option      = "error-no",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "debug.random-failure",
+          .voltype     = "debug/error-gen",
+          .option      = "random-failure",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+        { .key         = "debug.error-fops",
+          .voltype     = "debug/error-gen",
+          .option      = "enable",
+          .type        = NO_DOC,
+          .op_version  = 3
+        },
+
+
+        /* NFS xlator options */
+        { .key         = "nfs.enable-ino32",
+          .voltype     = "nfs/server",
+          .option      = "nfs.enable-ino32",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.mem-factor",
+          .voltype     = "nfs/server",
+          .option      = "nfs.mem-factor",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.export-dirs",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.export-dirs",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.export-volumes",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.export-volumes",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.addr-namelookup",
+          .voltype     = "nfs/server",
+          .option      = "rpc-auth.addr.namelookup",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.dynamic-volumes",
+          .voltype     = "nfs/server",
+          .option      = "nfs.dynamic-volumes",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.register-with-portmap",
+          .voltype     = "nfs/server",
+          .option      = "rpc.register-with-portmap",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.outstanding-rpc-limit",
+          .voltype     = "nfs/server",
+          .option      = "rpc.outstanding-rpc-limit",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.port",
+          .voltype     = "nfs/server",
+          .option      = "nfs.port",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-unix",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.auth-unix.*",
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-null",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.auth-null.*",
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-allow",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.addr.*.allow",
+          .op_version  = 1
+        },
+        { .key         = "nfs.rpc-auth-reject",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.addr.*.reject",
+          .op_version  = 1
+        },
+        { .key         = "nfs.ports-insecure",
+          .voltype     = "nfs/server",
+          .option      = "!rpc-auth.ports.*.insecure",
+          .op_version  = 1
+        },
+        { .key         = "nfs.transport-type",
+          .voltype     = "nfs/server",
+          .option      = "!nfs.transport-type",
+          .value       = "tcp",
+          .op_version  = 1,
+          .description = "Specifies the nfs transport type. Valid "
+                         "transport types are 'tcp' and 'rdma'."
+        },
+        { .key         = "nfs.trusted-sync",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.trusted-sync",
+          .op_version  = 1
+        },
+        { .key         = "nfs.trusted-write",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.trusted-write",
+          .op_version  = 1
+        },
+        { .key         = "nfs.volume-access",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.volume-access",
+          .op_version  = 1
+        },
+        { .key         = "nfs.export-dir",
+          .voltype     = "nfs/server",
+          .option      = "!nfs3.*.export-dir",
+          .op_version  = 1
+        },
+        { .key         = NFS_DISABLE_MAP_KEY,
+          .voltype     = "nfs/server",
+          .option      = "!nfs-disable",
+          .op_version  = 1
+        },
+        { .key         = "nfs.nlm",
+          .voltype     = "nfs/server",
+          .option      = "nfs.nlm",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.acl",
+          .voltype     = "nfs/server",
+          .option      = "nfs.acl",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.mount-udp",
+          .voltype     = "nfs/server",
+          .option      = "nfs.mount-udp",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.mount-rmtab",
+          .voltype     = "nfs/server",
+          .option      = "nfs.mount-rmtab",
+          .type        = GLOBAL_DOC,
+          .op_version  = 1
+        },
+        { .key         = "nfs.server-aux-gids",
+          .voltype     = "nfs/server",
+          .option      = "nfs.server-aux-gids",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "nfs.drc",
+          .voltype     = "nfs/server",
+          .option      = "nfs.drc",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.drc-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs.drc-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.read-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.read-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.write-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.write-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+        { .key         = "nfs.readdir-size",
+          .voltype     = "nfs/server",
+          .option      = "nfs3.readdir-size",
+          .type        = GLOBAL_DOC,
+          .op_version  = 3
+        },
+
+        /* Other options which don't fit any place above */
+        { .key        = "features.read-only",
+          .voltype    = "features/read-only",
+          .option     = "!read-only",
+          .value      = "off",
+          .op_version = 1,
+          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key        = "features.worm",
+          .voltype    = "features/worm",
+          .option     = "!worm",
+          .value      = "off",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
+        },
+        { .key         = "storage.linux-aio",
+          .voltype     = "storage/posix",
+          .op_version  = 1
+        },
+        { .key         = "storage.batch-fsync-mode",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.batch-fsync-delay-usec",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.owner-uid",
+          .voltype     = "storage/posix",
+          .option      = "brick-uid",
+          .op_version  = 1
+        },
+        { .key         = "storage.owner-gid",
+          .voltype     = "storage/posix",
+          .option      = "brick-gid",
+          .op_version  = 1
+        },
+        { .key         = "storage.node-uuid-pathinfo",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.health-check-interval",
+          .voltype     = "storage/posix",
+          .op_version  = 3
+        },
+        { .key         = "storage.bd-aio",
+          .voltype     = "storage/bd",
+          .op_version  = 3
+        },
+        { .key        = "config.memory-accounting",
+          .voltype    = "configuration",
+          .option     = "!config",
+          .op_version = 2,
+          .flags      = OPT_FLAG_CLIENT_OPT
+        },
+        { .key         = "config.transport",
+          .voltype     = "configuration",
+          .option      = "!config",
+          .op_version  = 2
+        },
+        { .key         = GLUSTERD_QUORUM_TYPE_KEY,
+          .voltype     = "mgmt/glusterd",
+          .value       = "off",
+          .op_version  = 2
+        },
+        { .key         = GLUSTERD_QUORUM_RATIO_KEY,
+          .voltype     = "mgmt/glusterd",
+          .value       = "0",
+          .op_version  = 2
+        },
+        /* changelog translator - global tunables */
+        { .key         = "changelog.changelog",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "changelog.changelog-dir",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "changelog.encoding",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "changelog.rollover-time",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = "changelog.fsync-interval",
+          .voltype     = "features/changelog",
+          .type        = NO_DOC,
+          .op_version  = 2
+        },
+        { .key         = NULL
+        }
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
new file mode 100644
index 000000000..59288ada0
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -0,0 +1,1597 @@
+/*
+   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <time.h>
+#include <grp.h>
+#include <sys/uio.h>
+#include <sys/resource.h>
+
+#include <libgen.h>
+#include "uuid.h"
+
+#include "glusterd.h"
+#include "rpcsvc.h"
+#include "fnmatch.h"
+#include "xlator.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "list.h"
+#include "dict.h"
+#include "compat.h"
+#include "compat-errno.h"
+#include "statedump.h"
+#include "glusterd-sm.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-store.h"
+#include "glusterd-hooks.h"
+#include "glusterd-utils.h"
+#include "glusterd-locks.h"
+#include "common-utils.h"
+#include "run.h"
+
+#include "syncop.h"
+
+#include "glusterd-mountbroker.h"
+
+extern struct rpcsvc_program gluster_handshake_prog;
+extern struct rpcsvc_program gluster_cli_getspec_prog;
+extern struct rpcsvc_program gluster_pmap_prog;
+extern glusterd_op_info_t opinfo;
+extern struct rpcsvc_program gd_svc_mgmt_prog;
+extern struct rpcsvc_program gd_svc_mgmt_v3_prog;
+extern struct rpcsvc_program gd_svc_peer_prog;
+extern struct rpcsvc_program gd_svc_cli_prog;
+extern struct rpcsvc_program gd_svc_cli_prog_ro;
+extern struct rpc_clnt_program gd_brick_prog;
+extern struct rpcsvc_program glusterd_mgmt_hndsk_prog;
+
+extern char snap_mount_folder[PATH_MAX];
+
+rpcsvc_cbk_program_t glusterd_cbk_prog = {
+        .progname  = "Gluster Callback",
+        .prognum   = GLUSTER_CBK_PROGRAM,
+        .progver   = GLUSTER_CBK_VERSION,
+};
+
+struct rpcsvc_program *gd_inet_programs[] = {
+        &gd_svc_peer_prog,
+        &gd_svc_cli_prog_ro,
+        &gd_svc_mgmt_prog,
+        &gd_svc_mgmt_v3_prog,
+        &gluster_pmap_prog,
+        &gluster_handshake_prog,
+        &glusterd_mgmt_hndsk_prog,
+};
+int gd_inet_programs_count = (sizeof (gd_inet_programs) /
+                              sizeof (gd_inet_programs[0]));
+
+struct rpcsvc_program *gd_uds_programs[] = {
+        &gd_svc_cli_prog,
+        &gluster_cli_getspec_prog,
+};
+int gd_uds_programs_count = (sizeof (gd_uds_programs) /
+                             sizeof (gd_uds_programs[0]));
+
+const char *gd_op_list[GD_OP_MAX + 1] = {
+        [GD_OP_NONE]                    = "Invalid op",
+        [GD_OP_CREATE_VOLUME]           = "Create",
+        [GD_OP_START_BRICK]             = "Start Brick",
+        [GD_OP_STOP_BRICK]              = "Stop Brick",
+        [GD_OP_DELETE_VOLUME]           = "Delete",
+        [GD_OP_START_VOLUME]            = "Start",
+        [GD_OP_STOP_VOLUME]             = "Stop",
+        [GD_OP_DEFRAG_VOLUME]           = "Rebalance",
+        [GD_OP_ADD_BRICK]               = "Add brick",
+        [GD_OP_REMOVE_BRICK]            = "Remove brick",
+        [GD_OP_REPLACE_BRICK]           = "Replace brick",
+        [GD_OP_SET_VOLUME]              = "Set",
+        [GD_OP_RESET_VOLUME]            = "Reset",
+        [GD_OP_SYNC_VOLUME]             = "Sync",
+        [GD_OP_LOG_ROTATE]              = "Log rotate",
+        [GD_OP_GSYNC_SET]               = "Geo-replication",
+        [GD_OP_PROFILE_VOLUME]          = "Profile",
+        [GD_OP_QUOTA]                   = "Quota",
+        [GD_OP_STATUS_VOLUME]           = "Status",
+        [GD_OP_REBALANCE]               = "Rebalance",
+        [GD_OP_HEAL_VOLUME]             = "Heal",
+        [GD_OP_STATEDUMP_VOLUME]        = "Statedump",
+        [GD_OP_LIST_VOLUME]             = "Lists",
+        [GD_OP_CLEARLOCKS_VOLUME]       = "Clear locks",
+        [GD_OP_DEFRAG_BRICK_VOLUME]     = "Rebalance",
+        [GD_OP_COPY_FILE]               = "Copy File",
+        [GD_OP_SYS_EXEC]                = "Execute system commands",
+        [GD_OP_GSYNC_CREATE]            = "Geo-replication Create",
+        [GD_OP_SNAP]                    = "Snapshot",
+        [GD_OP_MAX]                     = "Invalid op"
+};
+
+static int
+glusterd_opinfo_init ()
+{
+        int32_t ret = -1;
+
+        opinfo.op = GD_OP_NONE;
+
+        return ret;
+}
+
+
+int
+glusterd_uuid_init ()
+{
+        int             ret = -1;
+        xlator_t        *this = NULL;
+        glusterd_conf_t *priv = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+
+        ret = glusterd_retrieve_uuid ();
+        if (ret == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "retrieved UUID: %s", uuid_utoa (priv->uuid));
+                return 0;
+        }
+
+        ret = glusterd_uuid_generate_save ();
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                          "Unable to generate and save new UUID");
+                return ret;
+        }
+
+        return 0;
+}
+
+int
+glusterd_uuid_generate_save ()
+{
+        int               ret = -1;
+        glusterd_conf_t   *priv = NULL;
+        xlator_t          *this = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        uuid_generate (priv->uuid);
+
+        gf_log (this->name, GF_LOG_INFO, "generated UUID: %s",
+                uuid_utoa (priv->uuid));
+
+        ret = glusterd_store_global_info (this);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unable to store the generated uuid %s",
+                        uuid_utoa (priv->uuid));
+
+        return ret;
+}
+
+int
+glusterd_options_init (xlator_t *this)
+{
+        int             ret = -1;
+        glusterd_conf_t *priv = NULL;
+        char            *initial_version = "0";
+
+        priv = this->private;
+
+        priv->opts = dict_new ();
+        if (!priv->opts)
+                goto out;
+
+        ret = glusterd_store_retrieve_options (this);
+        if (ret == 0)
+                goto out;
+
+        ret = dict_set_str (priv->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                            initial_version);
+        if (ret)
+                goto out;
+        ret = glusterd_store_options (this, priv->opts);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to store version");
+                return ret;
+        }
+out:
+
+        return 0;
+}
+int
+glusterd_fetchspec_notify (xlator_t *this)
+{
+        int              ret   = -1;
+        glusterd_conf_t *priv  = NULL;
+        rpc_transport_t *trans = NULL;
+
+        priv = this->private;
+
+        pthread_mutex_lock (&priv->xprt_lock);
+        {
+                list_for_each_entry (trans, &priv->xprt_list, list) {
+                        rpcsvc_callback_submit (priv->rpc, trans,
+                                                &glusterd_cbk_prog,
+                                                GF_CBK_FETCHSPEC, NULL, 0);
+                }
+        }
+        pthread_mutex_unlock (&priv->xprt_lock);
+
+        ret = 0;
+
+        return ret;
+}
+
+int
+glusterd_priv (xlator_t *this)
+{
+        return 0;
+}
+
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_gld_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int
+glusterd_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                     void *data)
+{
+        xlator_t            *this = NULL;
+        rpc_transport_t     *xprt = NULL;
+        glusterd_conf_t     *priv = NULL;
+
+        if (!xl || !data) {
+                gf_log ("glusterd", GF_LOG_WARNING,
+                        "Calling rpc_notify without initializing");
+                goto out;
+        }
+
+        this = xl;
+        xprt = data;
+
+        priv = this->private;
+
+        switch (event) {
+        case RPCSVC_EVENT_ACCEPT:
+        {
+                INIT_LIST_HEAD (&xprt->list);
+
+                pthread_mutex_lock (&priv->xprt_lock);
+                list_add_tail (&xprt->list, &priv->xprt_list);
+                pthread_mutex_unlock (&priv->xprt_lock);
+                break;
+        }
+        case RPCSVC_EVENT_DISCONNECT:
+        {
+                pthread_mutex_lock (&priv->xprt_lock);
+                list_del (&xprt->list);
+                pthread_mutex_unlock (&priv->xprt_lock);
+                pmap_registry_remove (this, 0, NULL, GF_PMAP_PORT_NONE, xprt);
+                break;
+        }
+
+        default:
+                break;
+        }
+
+out:
+        return 0;
+}
+
+
+inline int32_t
+glusterd_program_register (xlator_t *this, rpcsvc_t *svc,
+                           rpcsvc_program_t *prog)
+{
+        int32_t ret = -1;
+
+        ret = rpcsvc_program_register (svc, prog);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "cannot register program (name: %s, prognum:%d, "
+                        "progver:%d)", prog->progname, prog->prognum,
+                        prog->progver);
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int
+glusterd_rpcsvc_options_build (dict_t *options)
+{
+        int             ret = 0;
+        uint32_t        backlog = 0;
+
+        ret = dict_get_uint32 (options, "transport.socket.listen-backlog",
+                               &backlog);
+
+        if (ret) {
+                backlog = GLUSTERD_SOCKET_LISTEN_BACKLOG;
+                ret = dict_set_uint32 (options,
+                                      "transport.socket.listen-backlog",
+                                      backlog);
+                if (ret)
+                        goto out;
+        }
+
+        gf_log ("", GF_LOG_DEBUG, "listen-backlog value: %d", backlog);
+
+out:
+        return ret;
+}
+
+#if SYNCDAEMON_COMPILE
+static int
+glusterd_check_gsync_present (int *valid_state)
+{
+        char                buff[PATH_MAX] = {0, };
+        runner_t            runner = {0,};
+        char               *ptr = NULL;
+        int                 ret = 0;
+
+        runinit (&runner);
+        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
+        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
+        ret = runner_start (&runner);
+        if (ret == -1) {
+                if (errno == ENOENT) {
+                        gf_log ("glusterd", GF_LOG_INFO, GEOREP
+                                 " module not installed in the system");
+                        *valid_state = 0;
+                }
+                else {
+                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP
+                                  " module not working as desired");
+                        *valid_state = -1;
+                }
+                goto out;
+        }
+
+        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
+        if (ptr) {
+                if (!strstr (buff, "gsyncd")) {
+                        ret = -1;
+                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
+                                 "working as desired");
+                        *valid_state = -1;
+                        goto out;
+                }
+        } else {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
+                         "working as desired");
+                *valid_state = -1;
+                goto out;
+        }
+
+        ret = 0;
+ out:
+
+        runner_end (&runner);
+
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+
+}
+
+static int
+group_write_allow (char *path, gid_t gid)
+{
+        struct stat st = {0,};
+        int ret        = 0;
+
+        ret = stat (path, &st);
+        if (ret == -1)
+                goto out;
+        GF_ASSERT (S_ISDIR (st.st_mode));
+
+        ret = chown (path, -1, gid);
+        if (ret == -1)
+                goto out;
+
+        ret = chmod (path, (st.st_mode & ~S_IFMT) | S_IWGRP|S_IXGRP|S_ISVTX);
+
+ out:
+        if (ret == -1)
+                gf_log ("", GF_LOG_CRITICAL,
+                        "failed to set up write access to %s for group %d (%s)",
+                        path, gid, strerror (errno));
+        return ret;
+}
+
+static int
+glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
+{
+        char *greplg_s   = NULL;
+        struct group *gr = NULL;
+        int ret          = 0;
+
+        GF_ASSERT (georepdir);
+        GF_ASSERT (conf);
+
+        if (strlen (conf->workdir)+2 > PATH_MAX-strlen(GEOREP)) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "directory path %s/"GEOREP" is longer than PATH_MAX",
+                        conf->workdir);
+                goto out;
+        }
+
+        snprintf (georepdir, PATH_MAX, "%s/"GEOREP, conf->workdir);
+        ret = mkdir_p (georepdir, 0777, _gf_true);
+        if (-1 == ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Unable to create "GEOREP" directory %s",
+                        georepdir);
+                goto out;
+        }
+
+        if (strlen (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP) >= PATH_MAX) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
+                        GEOREP" is longer than PATH_MAX");
+                goto out;
+        }
+        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP, 0777, _gf_true);
+        if (-1 == ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Unable to create "GEOREP" log directory");
+                goto out;
+        }
+
+        /* Slave log file directory */
+        if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves") >= PATH_MAX) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
+                        GEOREP"-slaves"" is longer than PATH_MAX");
+                goto out;
+        }
+        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves", 0777,
+                      _gf_true);
+        if (-1 == ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Unable to create "GEOREP" slave log directory");
+                goto out;
+        }
+
+        /* MountBroker log file directory */
+        if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr") >= PATH_MAX) {
+                ret = -1;
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP
+                        "-slaves/mbr"" is longer than PATH_MAX");
+                goto out;
+        }
+        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr", 0777,
+                      _gf_true);
+        if (-1 == ret) {
+                gf_log ("glusterd", GF_LOG_CRITICAL,
+                        "Unable to create "GEOREP" mountbroker slave log directory");
+                goto out;
+        }
+
+        ret = dict_get_str (THIS->options, GEOREP"-log-group", &greplg_s);
+        if (ret)
+                ret = 0;
+        else {
+                gr = getgrnam (greplg_s);
+                if (!gr) {
+                        gf_log ("glusterd", GF_LOG_CRITICAL,
+                                "group "GEOREP"-log-group %s does not exist", greplg_s);
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP,
+                                         gr->gr_gid);
+                if (ret == 0)
+                        ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
+                                                 GEOREP"-slaves", gr->gr_gid);
+                if (ret == 0)
+                        ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
+                                                 GEOREP"-slaves/mbr", gr->gr_gid);
+        }
+
+ out:
+        gf_log("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+static void
+runinit_gsyncd_setrx (runner_t *runner, glusterd_conf_t *conf)
+{
+        runinit (runner);
+        runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
+        runner_argprintf (runner, "%s/"GSYNC_CONF_TEMPLATE, conf->workdir);
+        runner_add_arg (runner, "--config-set-rx");
+}
+
+static int
+configure_syncdaemon (glusterd_conf_t *conf)
+#define RUN_GSYNCD_CMD do {                                                          \
+        ret = runner_run_reuse (&runner);                                            \
+        if (ret == -1) {                                                             \
+                runner_log (&runner, "glusterd", GF_LOG_ERROR, "command failed");    \
+                runner_end (&runner);                                                \
+                goto out;                                                            \
+        }                                                                            \
+        runner_end (&runner);                                                        \
+} while (0)
+{
+        int ret = 0;
+        runner_t runner = {0,};
+        char georepdir[PATH_MAX] = {0,};
+        int valid_state = 0;
+
+        ret = setenv ("_GLUSTERD_CALLED_", "1", 1);
+        if (ret < 0) {
+                ret = 0;
+                goto out;
+        }
+        valid_state = -1;
+        ret = glusterd_check_gsync_present (&valid_state);
+        if (-1 == ret) {
+                ret = valid_state;
+                goto out;
+        }
+
+        glusterd_crt_georep_folders (georepdir, conf);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        /************
+         * master pre-configuration
+         ************/
+
+        /* remote-gsyncd */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "remote-gsyncd", GSYNCD_PREFIX"/gsyncd", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
+                         ".", "^ssh:", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-command-dir */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "gluster-params",
+                         "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* ssh-command */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "ssh-command");
+        runner_argprintf (&runner,
+                          "ssh -oPasswordAuthentication=no "
+                           "-oStrictHostKeyChecking=no "
+                           "-i %s/secret.pem", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* pid-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "pid-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.pid", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.status", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-detail-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-detail-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+                          georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-detail-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-detail-file");
+        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+                          georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* state-socket */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg (&runner, "state-socket-unencoded");
+        runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.socket", georepdir);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* socketdir */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* log-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner,
+                         "log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.log",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-log-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner,
+                         "gluster-log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
+                         ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* ignore-deletes */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "ignore-deletes", "true", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* special-sync-mode */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* change-detector == changelog */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_arg(&runner, "working-dir");
+        runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+                         DEFAULT_VAR_RUN_DIRECTORY);
+        runner_add_args (&runner, ".", ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /************
+         * slave pre-configuration
+         ************/
+
+        /* gluster-command-dir */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-params */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner, "gluster-params",
+                         "aux-gfid-mount xlator-option=*-dht.assert-no-child-down=true",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* log-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner,
+                         "log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* MountBroker log-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner,
+                         "log-file-mbr",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+        /* gluster-log-file */
+        runinit_gsyncd_setrx (&runner, conf);
+        runner_add_args (&runner,
+                         "gluster-log-file",
+                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.gluster.log",
+                         ".", NULL);
+        RUN_GSYNCD_CMD;
+
+ out:
+        return ret ? -1 : 0;
+}
+#undef RUN_GSYNCD_CMD
+#else /* SYNCDAEMON_COMPILE */
+static int
+configure_syncdaemon (glusterd_conf_t *conf)
+{
+        return 0;
+}
+#endif /* !SYNCDAEMON_COMPILE */
+
+
+static int
+check_prepare_mountbroker_root (char *mountbroker_root)
+{
+        int dfd0        = -1;
+        int dfd         = -1;
+        int dfd2        = -1;
+        struct stat st  = {0,};
+        struct stat st2 = {0,};
+        int ret         = 0;
+
+        ret = open (mountbroker_root, O_RDONLY);
+        if (ret != -1) {
+                dfd = ret;
+                ret = fstat (dfd, &st);
+        }
+        if (ret == -1 || !S_ISDIR (st.st_mode)) {
+                gf_log ("", GF_LOG_ERROR,
+                        "cannot access mountbroker-root directory %s",
+                        mountbroker_root);
+                ret = -1;
+                goto out;
+        }
+        if (st.st_uid != 0 ||
+            (st.st_mode & (S_IWGRP|S_IWOTH))) {
+                gf_log ("", GF_LOG_ERROR,
+                        "permissions on mountbroker-root directory %s are "
+                        "too liberal", mountbroker_root);
+                ret = -1;
+                goto out;
+        }
+        if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
+                gf_log ("", GF_LOG_WARNING,
+                        "permissions on mountbroker-root directory %s are "
+                        "probably too strict", mountbroker_root);
+        }
+
+        dfd0 = dup (dfd);
+
+        for (;;) {
+                ret = openat (dfd, "..", O_RDONLY);
+                if (ret != -1) {
+                        dfd2 = ret;
+                        ret = fstat (dfd2, &st2);
+                }
+                if (ret == -1) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "error while checking mountbroker-root ancestors "
+                                "%d (%s)", errno, strerror (errno));
+                        goto out;
+                }
+
+                if (st2.st_ino == st.st_ino)
+                        break; /* arrived to root */
+
+                if (st2.st_uid != 0 ||
+                    ((st2.st_mode & (S_IWGRP|S_IWOTH)) &&
+                     !(st2.st_mode & S_ISVTX))) {
+                        gf_log ("", GF_LOG_ERROR,
+                                "permissions on ancestors of mountbroker-root "
+                                "directory are too liberal");
+                        ret = -1;
+                        goto out;
+                }
+                if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
+                        gf_log ("", GF_LOG_WARNING,
+                                "permissions on ancestors of mountbroker-root "
+                                "directory are probably too strict");
+                }
+
+                close (dfd);
+                dfd = dfd2;
+                st = st2;
+        }
+
+        ret = mkdirat (dfd0, MB_HIVE, 0711);
+        if (ret == -1 && errno == EEXIST)
+                ret = 0;
+        if (ret != -1)
+                ret = fstatat (dfd0, MB_HIVE, &st, AT_SYMLINK_NOFOLLOW);
+        if (ret == -1 || st.st_mode != (S_IFDIR|0711)) {
+                gf_log ("", GF_LOG_ERROR,
+                        "failed to set up mountbroker-root directory %s",
+                        mountbroker_root);
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        if (dfd0 != -1)
+                close (dfd0);
+        if (dfd != -1)
+                close (dfd);
+        if (dfd2 != -1)
+                close (dfd2);
+
+        return ret;
+}
+
+static int
+_install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
+{
+        glusterd_conf_t *priv           = THIS->private;
+        char            *label          = NULL;
+        gf_boolean_t     georep         = _gf_false;
+        gf_boolean_t     ghadoop        = _gf_false;
+        char            *pdesc          = value->data;
+        char            *volname        = NULL;
+        int              rv             = 0;
+        gf_mount_spec_t *mspec          = NULL;
+        char            *user           = NULL;
+        char            *volfile_server = NULL;
+
+        label = strtail (key, "mountbroker.");
+
+        /* check for presence of geo-rep/hadoop label */
+        if (!label) {
+                label = strtail (key, "mountbroker-"GEOREP".");
+                if (label)
+                        georep = _gf_true;
+                else {
+                        label = strtail (key, "mountbroker-"GHADOOP".");
+                        if (label)
+                                ghadoop = _gf_true;
+                }
+        }
+
+        if (!label)
+                return 0;
+
+        mspec = GF_CALLOC (1, sizeof (*mspec), gf_gld_mt_mount_spec);
+        if (!mspec)
+                goto err;
+        mspec->label = label;
+
+        if (georep || ghadoop) {
+                volname = gf_strdup (pdesc);
+                if (!volname)
+                        goto err;
+                user = strchr (volname, ':');
+                if (user) {
+                        *user = '\0';
+                        user++;
+                } else
+                        user = label;
+
+                if (georep)
+                        rv = make_georep_mountspec (mspec, volname, user);
+
+                if (ghadoop) {
+                        volfile_server = strchr (user, ':');
+                        if (volfile_server)
+                                *volfile_server++ = '\0';
+                        else
+                                volfile_server = "localhost";
+
+                        rv = make_ghadoop_mountspec (mspec, volname, user, volfile_server);
+                }
+
+                GF_FREE (volname);
+                if (rv != 0)
+                        goto err;
+        } else if (parse_mount_pattern_desc (mspec, pdesc) != 0)
+                goto err;
+
+        list_add_tail (&mspec->speclist, &priv->mount_specs);
+
+        return 0;
+ err:
+
+        gf_log ("", GF_LOG_ERROR,
+                "adding %smount spec failed: label: %s desc: %s",
+                georep ? GEOREP" " : "", label, pdesc);
+
+        return -1;
+}
+
+
+static int
+gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+        glusterd_conf_t     *priv = THIS->private;
+        synclock_unlock (&priv->big_lock);
+        return ret;
+}
+
+static void
+glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
+{
+        xlator_t        *this = NULL;
+        glusterd_conf_t *priv = NULL;
+        int             ret   = -1;
+
+        this = THIS;
+        priv = this->private;
+
+        synclock_lock (&priv->big_lock);
+        ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
+                            opaque);
+        if (ret)
+                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
+                        " and other volume related services");
+}
+
+int
+glusterd_uds_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                            void *data)
+{
+        /* glusterd_rpcsvc_notify() does stuff that calls coming in from the
+         * unix domain socket don't need. This is just an empty function to be
+         * used for the uds listener. This will be used later if required.
+         */
+        return 0;
+}
+
+/* The glusterd unix domain socket listener only listens for cli */
+rpcsvc_t *
+glusterd_init_uds_listener (xlator_t *this)
+{
+        int             ret = -1;
+        dict_t          *options = NULL;
+        rpcsvc_t        *rpc = NULL;
+        data_t          *sock_data = NULL;
+        char            sockfile[PATH_MAX+1] = {0,};
+        int             i = 0;
+
+
+        GF_ASSERT (this);
+
+        sock_data = dict_get (this->options, "glusterd-sockfile");
+        if (!sock_data) {
+                strncpy (sockfile, DEFAULT_GLUSTERD_SOCKFILE, PATH_MAX);
+        } else {
+                strncpy (sockfile, sock_data->data, PATH_MAX);
+        }
+
+        options = dict_new ();
+        if (!options)
+                goto out;
+
+        ret = rpcsvc_transport_unix_options_build (&options, sockfile);
+        if (ret)
+                goto out;
+
+        rpc = rpcsvc_init (this, this->ctx, options, 8);
+        if (rpc == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = rpcsvc_register_notify (rpc, glusterd_uds_rpcsvc_notify,
+                                      this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Failed to register notify function");
+                goto out;
+        }
+
+        ret = rpcsvc_create_listeners (rpc, options, this->name);
+        if (ret != 1) {
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to create listener");
+                goto out;
+        }
+        ret = 0;
+
+        for (i = 0; i < gd_uds_programs_count; i++) {
+                ret = glusterd_program_register (this, rpc, gd_uds_programs[i]);
+                if (ret) {
+                        i--;
+                        for (; i >= 0; i--)
+                                rpcsvc_program_unregister (rpc,
+                                                           gd_uds_programs[i]);
+
+                        goto out;
+                }
+        }
+
+out:
+        if (options)
+                dict_unref (options);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to start glusterd "
+                        "unix domain socket listener.");
+                if (rpc) {
+                        GF_FREE (rpc);
+                        rpc = NULL;
+                }
+        }
+        return rpc;
+}
+
+void
+glusterd_stop_uds_listener (xlator_t *this)
+{
+        glusterd_conf_t         *conf = NULL;
+        rpcsvc_listener_t       *listener = NULL;
+        rpcsvc_listener_t       *next = NULL;
+
+        GF_ASSERT (this);
+        conf = this->private;
+
+        (void) rpcsvc_program_unregister (conf->uds_rpc, &gd_svc_cli_prog);
+        (void) rpcsvc_program_unregister (conf->uds_rpc, &gluster_handshake_prog);
+
+        list_for_each_entry_safe (listener, next, &conf->uds_rpc->listeners,
+                                  list) {
+                rpcsvc_listener_destroy (listener);
+        }
+
+        (void) rpcsvc_unregister_notify (conf->uds_rpc, glusterd_rpcsvc_notify,
+                                         this);
+
+        unlink (DEFAULT_GLUSTERD_SOCKFILE);
+
+        GF_FREE (conf->uds_rpc);
+        conf->uds_rpc = NULL;
+
+        return;
+}
+
+static int
+glusterd_init_snap_folder (xlator_t *this)
+{
+        int             ret = -1;
+        struct stat     buf = {0,};
+
+        GF_ASSERT (this);
+
+        /* Snapshot volumes are mounted under /var/run/gluster/snaps folder.
+         * But /var/run is normally a symbolic link to /run folder, which
+         * creates problems as the entry point in the mtab for the mount point
+         * and glusterd maintained entry point will be different. Therefore
+         * identify the correct run folder and use it for snap volume mounting.
+         */
+        ret = lstat (GLUSTERD_VAR_RUN_DIR, &buf);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat fails on %s, exiting. (errno = %d)",
+                        GLUSTERD_VAR_RUN_DIR, errno);
+                goto out;
+        }
+
+        /* If /var/run is symlink then use /run folder */
+        if (S_ISLNK (buf.st_mode)) {
+                strcpy (snap_mount_folder, GLUSTERD_RUN_DIR);
+        } else {
+                strcpy (snap_mount_folder, GLUSTERD_VAR_RUN_DIR);
+        }
+
+        strcat (snap_mount_folder, GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
+
+        ret = stat (snap_mount_folder, &buf);
+        if ((ret != 0) && (ENOENT != errno)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat fails on %s, exiting. (errno = %d)",
+                        snap_mount_folder, errno);
+                ret = -1;
+                goto out;
+        }
+
+        if ((!ret) && (!S_ISDIR(buf.st_mode))) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Provided snap path %s is not a directory,"
+                        "exiting", snap_mount_folder);
+                ret = -1;
+                goto out;
+        }
+
+        if ((-1 == ret) && (ENOENT == errno)) {
+                /* Create missing folders */
+                ret = mkdir_p (snap_mount_folder, 0777, _gf_false);
+
+                if (-1 == ret) {
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to create directory %s"
+                                " ,errno = %d", snap_mount_folder, errno);
+                        goto out;
+                }
+        }
+
+out:
+        return ret;
+}
+
+/*
+ * init - called during glusterd initialization
+ *
+ * @this:
+ *
+ */
+int
+init (xlator_t *this)
+{
+        int32_t            ret               = -1;
+        rpcsvc_t          *rpc               = NULL;
+        rpcsvc_t          *uds_rpc           = NULL;
+        glusterd_conf_t   *conf              = NULL;
+        data_t            *dir_data          = NULL;
+        struct stat        buf               = {0,};
+        char               storedir [PATH_MAX] = {0,};
+        char               workdir [PATH_MAX] = {0,};
+        char               hooks_dir [PATH_MAX] = {0,};
+        char               cmd_log_filename [PATH_MAX] = {0,};
+        int                first_time        = 0;
+        char              *mountbroker_root  = NULL;
+        int                i                 = 0;
+        char              *valgrind_str      = NULL;
+
+        dir_data = dict_get (this->options, "working-directory");
+
+        if (!dir_data) {
+                //Use default working dir
+                strncpy (workdir, GLUSTERD_DEFAULT_WORKDIR, PATH_MAX);
+        } else {
+                strncpy (workdir, dir_data->data, PATH_MAX);
+        }
+
+        ret = stat (workdir, &buf);
+        if ((ret != 0) && (ENOENT != errno)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat fails on %s, exiting. (errno = %d)",
+                        workdir, errno);
+                exit (1);
+        }
+
+        if ((!ret) && (!S_ISDIR(buf.st_mode))) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Provided working area %s is not a directory,"
+                        "exiting", workdir);
+                exit (1);
+        }
+
+
+        if ((-1 == ret) && (ENOENT == errno)) {
+                ret = mkdir (workdir, 0777);
+
+                if (-1 == ret) {
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to create directory %s"
+                                " ,errno = %d", workdir, errno);
+                        exit (1);
+                }
+
+                first_time = 1;
+        }
+
+        setenv ("GLUSTERD_WORKING_DIR", workdir, 1);
+        gf_log (this->name, GF_LOG_INFO, "Using %s as working directory",
+                workdir);
+
+        ret = glusterd_init_snap_folder (this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Unable to create "
+                        "snap backend folder");
+                 exit (1);
+         }
+
+        snprintf (cmd_log_filename, PATH_MAX,"%s/.cmd_log_history",
+                  DEFAULT_LOG_FILE_DIRECTORY);
+        ret = gf_cmd_log_init (cmd_log_filename);
+
+        if (ret == -1) {
+                gf_log ("this->name", GF_LOG_CRITICAL,
+                        "Unable to create cmd log file %s", cmd_log_filename);
+                exit (1);
+        }
+
+        snprintf (storedir, PATH_MAX, "%s/vols", workdir);
+
+        ret = mkdir (storedir, 0777);
+
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create volume directory %s"
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
+        snprintf (storedir, PATH_MAX, "%s/peers", workdir);
+
+        ret = mkdir (storedir, 0777);
+
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create peers directory %s"
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
+        snprintf (storedir, PATH_MAX, "%s/bricks", DEFAULT_LOG_FILE_DIRECTORY);
+        ret = mkdir (storedir, 0777);
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create logs directory %s"
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
+        snprintf (storedir, PATH_MAX, "%s/nfs", workdir);
+        ret = mkdir (storedir, 0777);
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create nfs directory %s"
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
+        snprintf (storedir, PATH_MAX, "%s/glustershd", workdir);
+        ret = mkdir (storedir, 0777);
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create glustershd directory %s"
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
+        snprintf (storedir, PATH_MAX, "%s/groups", workdir);
+        ret = mkdir (storedir, 0777);
+        if ((-1 == ret) && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Unable to create glustershd directory %s"
+                        " ,errno = %d", storedir, errno);
+                exit (1);
+        }
+
+        ret = glusterd_rpcsvc_options_build (this->options);
+        if (ret)
+                goto out;
+        rpc = rpcsvc_init (this, this->ctx, this->options, 64);
+        if (rpc == NULL) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to init rpc");
+                goto out;
+        }
+
+        ret = rpcsvc_register_notify (rpc, glusterd_rpcsvc_notify, this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "rpcsvc_register_notify returned %d", ret);
+                goto out;
+        }
+
+        /*
+         * only one (atmost a pair - rdma and socket) listener for
+         * glusterd1_mop_prog, gluster_pmap_prog and gluster_handshake_prog.
+         */
+        ret = rpcsvc_create_listeners (rpc, this->options, this->name);
+        if (ret < 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "creation of listener failed");
+                ret = -1;
+                goto out;
+        }
+
+        for (i = 0; i < gd_inet_programs_count; i++) {
+                ret = glusterd_program_register (this, rpc,
+                                                 gd_inet_programs[i]);
+                if (ret) {
+                        i--;
+                        for (; i >= 0; i--)
+                                rpcsvc_program_unregister (rpc,
+                                                           gd_inet_programs[i]);
+
+                        goto out;
+                }
+        }
+
+        /* Start a unix domain socket listener just for cli commands
+         * This should prevent ports from being wasted by being in TIMED_WAIT
+         * when cli commands are done continuously
+         */
+        uds_rpc = glusterd_init_uds_listener (this);
+        if (uds_rpc == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        conf = GF_CALLOC (1, sizeof (glusterd_conf_t),
+                          gf_gld_mt_glusterd_conf_t);
+        GF_VALIDATE_OR_GOTO(this->name, conf, out);
+        conf->shd = GF_CALLOC (1, sizeof (nodesrv_t),
+                               gf_gld_mt_nodesrv_t);
+        GF_VALIDATE_OR_GOTO(this->name, conf->shd, out);
+        conf->nfs = GF_CALLOC (1, sizeof (nodesrv_t),
+                               gf_gld_mt_nodesrv_t);
+        GF_VALIDATE_OR_GOTO(this->name, conf->nfs, out);
+
+        INIT_LIST_HEAD (&conf->peers);
+        INIT_LIST_HEAD (&conf->volumes);
+        INIT_LIST_HEAD (&conf->snapshots);
+        INIT_LIST_HEAD (&conf->missed_snaps_list);
+
+        pthread_mutex_init (&conf->mutex, NULL);
+        conf->rpc = rpc;
+        conf->uds_rpc = uds_rpc;
+        conf->gfs_mgmt = &gd_brick_prog;
+        strncpy (conf->workdir, workdir, PATH_MAX);
+
+        synclock_init (&conf->big_lock);
+        pthread_mutex_init (&conf->xprt_lock, NULL);
+        INIT_LIST_HEAD (&conf->xprt_list);
+
+        glusterd_friend_sm_init ();
+        glusterd_op_sm_init ();
+        glusterd_opinfo_init ();
+        glusterd_mgmt_v3_lock_init ();
+        glusterd_txn_opinfo_dict_init ();
+        ret = glusterd_sm_tr_log_init (&conf->op_sm_log,
+                                       glusterd_op_sm_state_name_get,
+                                       glusterd_op_sm_event_name_get,
+                                       GLUSTERD_TR_LOG_SIZE);
+        if (ret)
+                goto out;
+
+         conf->base_port = GF_IANA_PRIV_PORTS_START;
+         if (dict_get_uint32(this->options, "base-port", &conf->base_port) == 0) {
+                 gf_log (this->name, GF_LOG_INFO,
+                         "base-port override: %d", conf->base_port);
+         }
+
+        /* Set option to run bricks on valgrind if enabled in glusterd.vol */
+        conf->valgrind = _gf_false;
+        ret = dict_get_str (this->options, "run-with-valgrind", &valgrind_str);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "cannot get run-with-valgrind value");
+        }
+        if (valgrind_str) {
+                if (gf_string2boolean (valgrind_str, &(conf->valgrind))) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "run-with-valgrind value not a boolean string");
+                }
+        }
+
+        this->private = conf;
+        (void) glusterd_nodesvc_set_online_status ("glustershd", _gf_false);
+
+        GLUSTERD_GET_HOOKS_DIR (hooks_dir, GLUSTERD_HOOK_VER, conf);
+        if (stat (hooks_dir, &buf)) {
+                ret = glusterd_hooks_create_hooks_directory (conf->workdir);
+                if (-1 == ret) {
+                        gf_log (this->name, GF_LOG_CRITICAL,
+                                "Unable to create hooks directory ");
+                        exit (1);
+                }
+        }
+
+        INIT_LIST_HEAD (&conf->mount_specs);
+
+        ret = dict_foreach (this->options, _install_mount_spec, NULL);
+        if (ret)
+                goto out;
+        ret = dict_get_str (this->options, "mountbroker-root",
+                            &mountbroker_root);
+        if (ret)
+                ret = 0;
+        else
+                ret = check_prepare_mountbroker_root (mountbroker_root);
+        if (ret)
+                goto out;
+
+        ret = configure_syncdaemon (conf);
+        if (ret)
+                goto out;
+
+        ret = glusterd_restore ();
+        if (ret < 0)
+                goto out;
+
+        /* If there are no 'friends', this would be the best time to
+         * spawn process/bricks that may need (re)starting since last
+         * time (this) glusterd was up.*/
+
+        if (list_empty (&conf->peers)) {
+                glusterd_launch_synctask (glusterd_spawn_daemons, NULL);
+        }
+        ret = glusterd_options_init (this);
+        if (ret < 0)
+                goto out;
+
+        ret = glusterd_handle_upgrade_downgrade (this->options, conf);
+        if (ret)
+                goto out;
+
+        ret = glusterd_hooks_spawn_worker (this);
+        if (ret)
+                goto out;
+
+        ret = 0;
+out:
+        if (ret < 0) {
+                if (this->private != NULL) {
+                        GF_FREE (this->private);
+                        this->private = NULL;
+                }
+
+        }
+
+        return ret;
+}
+
+
+
+
+/*
+ * fini - finish function for glusterd, called before
+ *        unloading gluster.
+ *
+ * @this:
+ *
+ */
+void
+fini (xlator_t *this)
+{
+        glusterd_conf_t *conf = NULL;
+        if (!this || !this->private)
+                goto out;
+
+        conf = this->private;
+
+        glusterd_stop_uds_listener (this);
+
+        FREE (conf->pmap);
+        if (conf->handle)
+                gf_store_handle_destroy (conf->handle);
+        glusterd_sm_tr_log_delete (&conf->op_sm_log);
+        glusterd_mgmt_v3_lock_fini ();
+        glusterd_txn_opinfo_dict_fini ();
+        GF_FREE (conf);
+
+        this->private = NULL;
+out:
+        return;
+}
+
+/*
+ * notify - notify function for glusterd
+ * @this:
+ * @trans:
+ * @event:
+ *
+ */
+int
+notify (xlator_t *this, int32_t event, void *data, ...)
+{
+        int          ret = 0;
+
+        switch (event) {
+                case GF_EVENT_POLLIN:
+                        break;
+
+                case GF_EVENT_POLLERR:
+                        break;
+
+                case GF_EVENT_TRANSPORT_CLEANUP:
+                        break;
+
+                default:
+                        default_notify (this, event, data);
+                        break;
+
+        }
+
+        return ret;
+}
+
+
+struct xlator_fops fops;
+
+struct xlator_cbks cbks;
+
+struct xlator_dumpops dumpops = {
+        .priv  = glusterd_priv,
+};
+
+
+struct volume_options options[] = {
+        { .key   = {"working-directory"},
+          .type  = GF_OPTION_TYPE_PATH,
+        },
+        { .key   = {"transport-type"},
+          .type  = GF_OPTION_TYPE_ANY,
+        },
+        { .key   = {"transport.*"},
+          .type  = GF_OPTION_TYPE_ANY,
+        },
+        { .key   = {"rpc-auth.*"},
+          .type  = GF_OPTION_TYPE_ANY,
+        },
+        { .key   = {"rpc-auth-allow-insecure"},
+          .type  = GF_OPTION_TYPE_BOOL,
+        },
+        { .key  = {"upgrade"},
+          .type = GF_OPTION_TYPE_BOOL,
+        },
+        { .key  = {"downgrade"},
+          .type = GF_OPTION_TYPE_BOOL,
+        },
+        { .key = {"bind-insecure"},
+          .type = GF_OPTION_TYPE_BOOL,
+        },
+        { .key  = {"mountbroker-root"},
+          .type = GF_OPTION_TYPE_PATH,
+        },
+        { .key  = {"mountbroker.*"},
+          .type = GF_OPTION_TYPE_ANY,
+        },
+        { .key  = {"mountbroker-"GEOREP".*"},
+          .type = GF_OPTION_TYPE_ANY,
+        },
+        { .key  = {"mountbroker-"GHADOOP".*"},
+          .type = GF_OPTION_TYPE_ANY,
+        },
+        { .key = {GEOREP"-log-group"},
+          .type = GF_OPTION_TYPE_ANY,
+        },
+        { .key = {"run-with-valgrind"},
+          .type = GF_OPTION_TYPE_BOOL,
+        },
+        { .key = {"server-quorum-type"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = { "none", "server"},
+          .description = "If set to server, enables the specified "
+          "volume to participate in quorum."
+        },
+        { .key = {"server-quorum-ratio"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .description = "Sets the quorum percentage for the trusted "
+          "storage pool."
+        },
+        { .key = {"glusterd-sockfile"},
+          .type = GF_OPTION_TYPE_PATH,
+          .description = "The socket file on which glusterd should listen for "
+                        "cli requests. Default is "DEFAULT_GLUSTERD_SOCKFILE "."
+        },
+        { .key = {"base-port"},
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Sets the base port for portmap query"
+        },
+        { .key = {"snap-brick-path"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "directory where the bricks for the snapshots will be created"
+        },
+        { .key   = {NULL} },
+};
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
new file mode 100644
index 000000000..3aa395ebc
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -0,0 +1,952 @@
+/*
+   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_H_
+#define _GLUSTERD_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+#include <dirent.h>
+#include <pthread.h>
+#include <libgen.h>
+
+#include "uuid.h"
+
+#include "rpc-clnt.h"
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "call-stub.h"
+#include "fd.h"
+#include "byte-order.h"
+#include "glusterd-mem-types.h"
+#include "rpcsvc.h"
+#include "glusterd-sm.h"
+#include "glusterd1-xdr.h"
+#include "protocol-common.h"
+#include "glusterd-pmap.h"
+#include "cli1-xdr.h"
+#include "syncop.h"
+#include "store.h"
+
+#define GLUSTERD_MAX_VOLUME_NAME        1000
+#define GLUSTERD_TR_LOG_SIZE            50
+#define GLUSTERD_NAME                   "glusterd"
+#define GLUSTERD_SOCKET_LISTEN_BACKLOG  128
+#define GLUSTERD_QUORUM_TYPE_KEY        "cluster.server-quorum-type"
+#define GLUSTERD_QUORUM_RATIO_KEY       "cluster.server-quorum-ratio"
+#define GLUSTERD_GLOBAL_OPT_VERSION     "global-option-version"
+#define GLUSTERD_COMMON_PEM_PUB_FILE    "/geo-replication/common_secret.pem.pub"
+#define GEO_CONF_MAX_OPT_VALS           5
+#define GLUSTERD_CREATE_HOOK_SCRIPT     "/hooks/1/gsync-create/post/" \
+                                        "S56glusterd-geo-rep-create-post.sh"
+
+
+#define GLUSTERD_SNAPS_MAX_HARD_LIMIT 256
+#define GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT 90
+#define GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT 100
+#define GLUSTERD_SERVER_QUORUM "server"
+
+#define FMTSTR_CHECK_VOL_EXISTS "Volume %s does not exist"
+#define FMTSTR_RESOLVE_BRICK "Could not find peer on which brick %s:%s resides"
+
+#define LOGSTR_FOUND_BRICK  "Found brick %s:%s in volume %s"
+#define LOGSTR_BUILD_PAYLOAD "Failed to build payload for operation 'Volume %s'"
+#define LOGSTR_STAGE_FAIL "Staging of operation 'Volume %s' failed on %s %s %s"
+#define LOGSTR_COMMIT_FAIL "Commit of operation 'Volume %s' failed on %s %s %s"
+
+#define OPERRSTR_BUILD_PAYLOAD "Failed to build payload. Please check the log "\
+                               "file for more details."
+#define OPERRSTR_STAGE_FAIL "Staging failed on %s. Please check the log file " \
+                            "for more details."
+#define OPERRSTR_COMMIT_FAIL "Commit failed on %s. Please check the log file "\
+                             "for more details."
+
+struct glusterd_volinfo_;
+typedef struct glusterd_volinfo_ glusterd_volinfo_t;
+
+struct glusterd_snap_;
+typedef struct glusterd_snap_ glusterd_snap_t;
+
+typedef enum glusterd_op_ {
+        GD_OP_NONE = 0,
+        GD_OP_CREATE_VOLUME,
+        GD_OP_START_BRICK,
+        GD_OP_STOP_BRICK,
+        GD_OP_DELETE_VOLUME,
+        GD_OP_START_VOLUME,
+        GD_OP_STOP_VOLUME,
+        GD_OP_DEFRAG_VOLUME,
+        GD_OP_ADD_BRICK,
+        GD_OP_REMOVE_BRICK,
+        GD_OP_REPLACE_BRICK,
+        GD_OP_SET_VOLUME,
+        GD_OP_RESET_VOLUME,
+        GD_OP_SYNC_VOLUME,
+        GD_OP_LOG_ROTATE,
+        GD_OP_GSYNC_SET,
+        GD_OP_PROFILE_VOLUME,
+        GD_OP_QUOTA,
+        GD_OP_STATUS_VOLUME,
+        GD_OP_REBALANCE,
+        GD_OP_HEAL_VOLUME,
+        GD_OP_STATEDUMP_VOLUME,
+        GD_OP_LIST_VOLUME,
+        GD_OP_CLEARLOCKS_VOLUME,
+        GD_OP_DEFRAG_BRICK_VOLUME,
+        GD_OP_COPY_FILE,
+        GD_OP_SYS_EXEC,
+        GD_OP_GSYNC_CREATE,
+        GD_OP_SNAP,
+        GD_OP_MAX,
+} glusterd_op_t;
+
+extern const char * gd_op_list[];
+
+struct glusterd_volgen {
+        dict_t *dict;
+};
+
+typedef struct {
+        struct rpc_clnt  *rpc;
+        gf_boolean_t      online;
+} nodesrv_t;
+
+typedef struct {
+        gf_boolean_t    quorum;
+        double          quorum_ratio;
+        uint64_t        gl_opt_version;
+} gd_global_opts_t;
+
+typedef struct {
+        struct _volfile_ctx      *volfile;
+        pthread_mutex_t           mutex;
+        struct list_head          peers;
+        struct list_head          xaction_peers;
+        gf_boolean_t              verify_volfile_checksum;
+        gf_boolean_t              trace;
+        uuid_t                    uuid;
+        char                      workdir[PATH_MAX];
+        rpcsvc_t                 *rpc;
+        nodesrv_t                *shd;
+        nodesrv_t                *nfs;
+        struct pmap_registry     *pmap;
+        struct list_head          volumes;
+        struct list_head          snapshots; /*List of snap volumes */
+        pthread_mutex_t           xprt_lock;
+        struct list_head          xprt_list;
+        gf_store_handle_t        *handle;
+        gf_timer_t               *timer;
+        glusterd_sm_tr_log_t      op_sm_log;
+        struct rpc_clnt_program  *gfs_mgmt;
+
+        struct list_head           mount_specs;
+        gf_boolean_t               valgrind;
+        pthread_t                  brick_thread;
+        void                      *hooks_priv;
+        /* need for proper handshake_t */
+        int                        op_version; /* Starts with 1 for 3.3.0 */
+        xlator_t                  *xl;  /* Should be set to 'THIS' before creating thread */
+        gf_boolean_t               pending_quorum_action;
+        dict_t                    *opts;
+        synclock_t                 big_lock;
+        gf_boolean_t               restart_done;
+        rpcsvc_t                  *uds_rpc; /* RPCSVC for the unix domain socket */
+        uint32_t                   base_port;
+        uint64_t                   snap_max_hard_limit;
+        uint64_t                   snap_max_soft_limit;
+        char                      *snap_bricks_directory;
+        gf_store_handle_t         *missed_snaps_list_shandle;
+        struct list_head           missed_snaps_list;
+} glusterd_conf_t;
+
+
+typedef enum gf_brick_status {
+        GF_BRICK_STOPPED,
+        GF_BRICK_STARTED,
+} gf_brick_status_t;
+
+struct glusterd_brickinfo {
+        char               hostname[1024];
+        char               path[PATH_MAX];
+        char               device_path[PATH_MAX];
+        struct list_head   brick_list;
+        uuid_t             uuid;
+        int                port;
+        int                rdma_port;
+        char              *logfile;
+        gf_boolean_t       signed_in;
+        gf_store_handle_t *shandle;
+        gf_brick_status_t  status;
+        struct rpc_clnt   *rpc;
+        int                decommissioned;
+        char vg[PATH_MAX]; /* FIXME: Use max size for length of vg */
+        int     caps; /* Capability */
+        int32_t            snap_status;
+};
+
+typedef struct glusterd_brickinfo glusterd_brickinfo_t;
+
+struct gf_defrag_brickinfo_ {
+        char *name;
+        int   files;
+        int   size;
+};
+
+typedef int (*defrag_cbk_fn_t) (glusterd_volinfo_t *volinfo,
+                                gf_defrag_status_t status);
+
+struct glusterd_defrag_info_ {
+        uint64_t                     total_files;
+        uint64_t                     total_data;
+        uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
+        gf_lock_t                    lock;
+        int                          cmd;
+        pthread_t                    th;
+        gf_defrag_status_t           defrag_status;
+        struct rpc_clnt             *rpc;
+        uint32_t                     connected;
+        char                         mount[1024];
+        char                         databuf[131072];
+        struct gf_defrag_brickinfo_ *bricks; /* volinfo->brick_count */
+
+        defrag_cbk_fn_t              cbk_fn;
+};
+
+
+typedef struct glusterd_defrag_info_ glusterd_defrag_info_t;
+
+typedef enum gf_transport_type_ {
+        GF_TRANSPORT_TCP,       //DEFAULT
+        GF_TRANSPORT_RDMA,
+        GF_TRANSPORT_BOTH_TCP_RDMA,
+} gf_transport_type;
+
+#define GF_DEFAULT_NFS_TRANSPORT  GF_TRANSPORT_RDMA
+
+typedef enum gf_rb_status_ {
+        GF_RB_STATUS_NONE,
+        GF_RB_STATUS_STARTED,
+        GF_RB_STATUS_PAUSED,
+} gf_rb_status_t;
+
+struct _auth {
+        char       *username;
+        char       *password;
+};
+
+typedef struct _auth auth_t;
+
+/* Capabilities of xlator */
+#define CAPS_BD               0x00000001
+#define CAPS_THIN             0x00000002
+#define CAPS_OFFLOAD_COPY     0x00000004
+#define CAPS_OFFLOAD_SNAPSHOT 0x00000008
+
+struct glusterd_rebalance_ {
+        gf_defrag_status_t       defrag_status;
+        uint64_t                 rebalance_files;
+        uint64_t                 rebalance_data;
+        uint64_t                 lookedup_files;
+        uint64_t                 skipped_files;
+        glusterd_defrag_info_t  *defrag;
+        gf_cli_defrag_type       defrag_cmd;
+        uint64_t                 rebalance_failures;
+        uuid_t                   rebalance_id;
+        double                   rebalance_time;
+        glusterd_op_t            op;
+        dict_t                  *dict; /* Dict to store misc information
+                                        * like list of bricks being removed */
+};
+
+typedef struct glusterd_rebalance_ glusterd_rebalance_t;
+
+struct glusterd_replace_brick_ {
+        gf_rb_status_t          rb_status;
+        glusterd_brickinfo_t   *src_brick;
+        glusterd_brickinfo_t   *dst_brick;
+        uuid_t                  rb_id;
+};
+
+typedef struct glusterd_replace_brick_ glusterd_replace_brick_t;
+
+struct glusterd_volinfo_ {
+        gf_lock_t                 lock;
+        char                      volname[GLUSTERD_MAX_VOLUME_NAME];
+        gf_boolean_t              is_snap_volume;
+        glusterd_snap_t          *snapshot;
+        gf_boolean_t              is_volume_restored;
+        char                      parent_volname[GLUSTERD_MAX_VOLUME_NAME];
+                                         /* In case of a snap volume
+                                            i.e (is_snap_volume == TRUE) this
+                                            field will contain the name of
+                                            the volume which is snapped. In
+                                            case of a non-snap volume, this
+                                            field will be initialized as N/A */
+        int                       type;
+        int                       brick_count;
+        uint64_t                  snap_count;
+        uint64_t                  snap_max_hard_limit;
+        struct list_head          vol_list;
+                                      /* In case of a snap volume
+                                         i.e (is_snap_volume == TRUE) this
+                                         is linked to glusterd_snap_t->volumes.
+                                         In case of a non-snap volume, this is
+                                         linked to glusterd_conf_t->volumes */
+        struct list_head          snapvol_list;
+                                      /* This is a current pointer for
+                                         glusterd_volinfo_t->snap_volumes */
+        struct list_head          bricks;
+        struct list_head          snap_volumes;
+                                      /* TODO : Need to remove this, as this
+                                       * is already part of snapshot object.
+                                       */
+        glusterd_volume_status    status;
+        int                       sub_count;  /* backward compatibility */
+        int                       stripe_count;
+        int                       replica_count;
+        int                       subvol_count; /* Number of subvolumes in a
+                                                 distribute volume */
+        int                       dist_leaf_count; /* Number of bricks in one
+                                                    distribute subvolume */
+        int                       port;
+        gf_store_handle_t        *shandle;
+        gf_store_handle_t        *rb_shandle;
+        gf_store_handle_t        *node_state_shandle;
+
+        /* Defrag/rebalance related */
+        glusterd_rebalance_t      rebal;
+
+        /* Replace brick status */
+        glusterd_replace_brick_t  rep_brick;
+
+        int                       version;
+        uint32_t                  cksum;
+        gf_transport_type         transport_type;
+        gf_transport_type         nfs_transport_type;
+
+        dict_t                   *dict;
+
+        uuid_t                    volume_id;
+        auth_t                    auth;
+        char                     *logdir;
+
+        dict_t                   *gsync_slaves;
+
+        int                       decommission_in_progress;
+        xlator_t                 *xl;
+
+        gf_boolean_t              memory_accounting;
+        int                      caps; /* Capability */
+
+        int                       op_version;
+        int                       client_op_version;
+};
+
+typedef enum gd_snap_status_ {
+        GD_SNAP_STATUS_NONE,
+        GD_SNAP_STATUS_INIT,
+        GD_SNAP_STATUS_IN_USE,
+        GD_SNAP_STATUS_DECOMMISSION,
+        GD_SNAP_STATUS_RESTORED,
+} gd_snap_status_t;
+
+struct glusterd_snap_ {
+        gf_lock_t                lock;
+        struct  list_head        volumes;
+        struct  list_head        snap_list;
+        char                     snapname[GLUSTERD_MAX_SNAP_NAME];
+        uuid_t                   snap_id;
+        char                    *description;
+        time_t                   time_stamp;
+        gf_boolean_t             snap_restored;
+        gd_snap_status_t         snap_status;
+        gf_store_handle_t       *shandle;
+};
+
+typedef struct glusterd_snap_op_ {
+        int32_t                brick_num;
+        char                  *brick_path;
+        int32_t                op;
+        int32_t                status;
+        struct list_head       snap_ops_list;
+} glusterd_snap_op_t;
+
+typedef struct glusterd_missed_snap_ {
+        char                   *node_snap_info;
+        struct list_head        missed_snaps;
+        struct list_head        snap_ops;
+} glusterd_missed_snap_info;
+
+typedef enum gd_node_type_ {
+        GD_NODE_NONE,
+        GD_NODE_BRICK,
+        GD_NODE_SHD,
+        GD_NODE_REBALANCE,
+        GD_NODE_NFS,
+} gd_node_type;
+
+typedef enum missed_snap_stat {
+        GD_MISSED_SNAP_NONE,
+        GD_MISSED_SNAP_PENDING,
+        GD_MISSED_SNAP_DONE,
+} missed_snap_stat;
+
+typedef struct glusterd_pending_node_ {
+        struct list_head list;
+        void   *node;
+        gd_node_type type;
+        int32_t index;
+} glusterd_pending_node_t;
+
+struct gsync_config_opt_vals_ {
+        char           *op_name;
+        int             no_of_pos_vals;
+        gf_boolean_t    case_sensitive;
+        char           *values[GEO_CONF_MAX_OPT_VALS];
+};
+
+enum glusterd_op_ret {
+        GLUSTERD_CONNECTION_AWAITED = 100,
+};
+
+enum glusterd_vol_comp_status_ {
+        GLUSTERD_VOL_COMP_NONE = 0,
+        GLUSTERD_VOL_COMP_SCS = 1,
+        GLUSTERD_VOL_COMP_UPDATE_REQ,
+        GLUSTERD_VOL_COMP_RJT,
+};
+
+#define GLUSTERD_DEFAULT_WORKDIR "/var/lib/glusterd"
+#define GLUSTERD_DEFAULT_PORT    GF_DEFAULT_BASE_PORT
+#define GLUSTERD_INFO_FILE      "glusterd.info"
+#define GLUSTERD_VOLUME_DIR_PREFIX "vols"
+#define GLUSTERD_PEER_DIR_PREFIX "peers"
+#define GLUSTERD_VOLUME_INFO_FILE "info"
+#define GLUSTERD_SNAP_INFO_FILE "info"
+#define GLUSTERD_VOLUME_RBSTATE_FILE "rbstate"
+#define GLUSTERD_BRICK_INFO_DIR "bricks"
+#define GLUSTERD_CKSUM_FILE "cksum"
+#define GLUSTERD_TRASH "trash"
+#define GLUSTERD_NODE_STATE_FILE "node_state.info"
+#define GLUSTERD_MISSED_SNAPS_LIST_FILE "missed_snaps_list"
+#define GLUSTERD_VOL_SNAP_DIR_PREFIX "snaps"
+
+#define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR        "/gluster/snaps"
+#define GLUSTERD_VAR_RUN_DIR                    "/var/run"
+#define GLUSTERD_RUN_DIR                        "/run"
+
+/* definitions related to replace brick */
+#define RB_CLIENT_MOUNTPOINT    "rb_mount"
+#define RB_CLIENTVOL_FILENAME   "rb_client.vol"
+#define RB_DSTBRICK_PIDFILE     "rb_dst_brick.pid"
+#define RB_DSTBRICKVOL_FILENAME "rb_dst_brick.vol"
+#define RB_PUMP_DEF_ARG         "default"
+
+#define GLUSTERD_UUID_LEN 50
+
+typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
+
+#define GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv)                       \
+        if (volinfo->is_snap_volume) {                                     \
+                snprintf (path, PATH_MAX, "%s/snaps/%s/%s", priv->workdir, \
+                          volinfo->snapshot->snapname, volinfo->volname);  \
+        } else {                                                           \
+                snprintf (path, PATH_MAX, "%s/vols/%s", priv->workdir,     \
+                          volinfo->volname);                               \
+        }
+
+#define GLUSTERD_GET_SNAP_DIR(path, snap, priv)                           \
+                snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,   \
+                          snap->snapname);
+
+#define GLUSTERD_GET_BRICK_DIR(path, volinfo, priv)                           \
+        if (volinfo->is_snap_volume) {                                        \
+                snprintf (path, PATH_MAX, "%s/snaps/%s/%s/%s", priv->workdir, \
+                          volinfo->snapshot->snapname, volinfo->volname,      \
+                          GLUSTERD_BRICK_INFO_DIR);                           \
+        } else {                                                              \
+                snprintf (path, PATH_MAX, "%s/%s/%s/%s", priv->workdir,       \
+                          GLUSTERD_VOLUME_DIR_PREFIX, volinfo->volname,       \
+                          GLUSTERD_BRICK_INFO_DIR);                           \
+        }
+
+#define GLUSTERD_GET_NFS_DIR(path, priv) \
+        snprintf (path, PATH_MAX, "%s/nfs", priv->workdir);
+
+#define GLUSTERD_REMOVE_SLASH_FROM_PATH(path,string) do {               \
+                int i = 0;                                              \
+                for (i = 1; i < strlen (path); i++) {                   \
+                        string[i-1] = path[i];                          \
+                        if (string[i-1] == '/')                         \
+                                string[i-1] = '-';                      \
+                }                                                       \
+        } while (0)
+
+#define GLUSTERD_GET_BRICK_PIDFILE(pidfile,volinfo,brickinfo, priv) do {      \
+                char exp_path[PATH_MAX] = {0,};                               \
+                char volpath[PATH_MAX]  = {0,};                               \
+                GLUSTERD_GET_VOLUME_DIR (volpath, volinfo, priv);             \
+                GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);  \
+                snprintf (pidfile, PATH_MAX, "%s/run/%s-%s.pid",              \
+                          volpath, brickinfo->hostname, exp_path);      \
+        } while (0)
+
+#define GLUSTERD_GET_NFS_PIDFILE(pidfile,nfspath) {                     \
+                snprintf (pidfile, PATH_MAX, "%s/run/nfs.pid",          \
+                          nfspath);                                     \
+        }
+
+#define GLUSTERD_STACK_DESTROY(frame) do {\
+                frame->local = NULL;                                    \
+                STACK_DESTROY (frame->root);                            \
+        } while (0)
+
+#define GLUSTERD_GET_DEFRAG_DIR(path, volinfo, priv) do {               \
+                char vol_path[PATH_MAX];                                \
+                GLUSTERD_GET_VOLUME_DIR(vol_path, volinfo, priv);       \
+                snprintf (path, PATH_MAX, "%s/rebalance",vol_path);     \
+        } while (0)
+
+#define GLUSTERD_GET_DEFRAG_SOCK_FILE(path, volinfo, priv) do {         \
+                char defrag_path[PATH_MAX];                             \
+                GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);    \
+                snprintf (path, PATH_MAX, "%s/%s.sock", defrag_path,    \
+                           uuid_utoa(MY_UUID));                         \
+        } while (0)
+
+#define GLUSTERD_GET_DEFRAG_PID_FILE(path, volinfo, priv) do {          \
+                char defrag_path[PATH_MAX];                             \
+                GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);    \
+                snprintf (path, PATH_MAX, "%s/%s.pid", defrag_path,     \
+                           uuid_utoa(MY_UUID));                         \
+        } while (0)
+
+#define GLUSTERD_GET_UUID_NOHYPHEN(ret_string, uuid) do {               \
+                char *snap_volname_ptr = ret_string;                    \
+                char *snap_volid_ptr = uuid_utoa(uuid);                 \
+                while (*snap_volid_ptr) {                               \
+                        if (*snap_volid_ptr == '-') {                   \
+                                snap_volid_ptr++;                       \
+                        } else {                                        \
+                                (*snap_volname_ptr++) =                 \
+                                (*snap_volid_ptr++);                    \
+                        }                                               \
+                }                                                       \
+                *snap_volname_ptr = '\0';                               \
+        } while (0)
+
+int glusterd_uuid_init();
+
+int glusterd_uuid_generate_save ();
+
+#define MY_UUID (__glusterd_uuid())
+
+static inline unsigned char *
+__glusterd_uuid()
+{
+        glusterd_conf_t *priv = THIS->private;
+
+        if (uuid_is_null (priv->uuid))
+                glusterd_uuid_init();
+        return &priv->uuid[0];
+}
+
+int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
+                                rpc_clnt_event_t event,
+                                void *data, rpc_clnt_notify_t notify_fn);
+
+int
+glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
+                         int count, void *myframe, fop_cbk_fn_t fn);
+
+int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn);
+
+int32_t
+glusterd_brick_from_brickinfo (glusterd_brickinfo_t *brickinfo,
+                               char **new_brick);
+int
+glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
+                      dict_t *dict);
+
+int
+glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
+                               char *remote_hostname, int port, int32_t op_ret,
+                               int32_t op_errno);
+
+int
+glusterd_friend_find (uuid_t uuid, char *hostname,
+                      glusterd_peerinfo_t **peerinfo);
+
+int
+glusterd_friend_add (const char *hoststr, int port,
+                     glusterd_friend_sm_state_t state,
+                     uuid_t *uuid, glusterd_peerinfo_t **friend,
+                     gf_boolean_t restore, glusterd_peerctx_args_t *args);
+
+int
+glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                            glusterd_peerctx_args_t *args);
+int
+glusterd_friend_remove (uuid_t uuid, char *hostname);
+
+int
+glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status);
+
+int
+glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status);
+
+int
+glusterd_op_mgmt_v3_lock_send_resp (rpcsvc_request_t *req,
+                                    uuid_t *txn_id, int32_t status);
+
+int
+glusterd_op_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req,
+                                      uuid_t *txn_id, int32_t status);
+
+int
+glusterd_op_stage_send_resp (rpcsvc_request_t *req,
+                             int32_t op, int32_t status,
+                             char *op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_commmit_send_resp (rpcsvc_request_t *req,
+                               int32_t op, int32_t status);
+
+int32_t
+glusterd_create_volume (rpcsvc_request_t *req, dict_t *dict);
+
+int
+glusterd_handle_incoming_friend_req (rpcsvc_request_t *req);
+
+int
+glusterd_handle_probe_query (rpcsvc_request_t *req);
+
+int
+glusterd_handle_cluster_lock (rpcsvc_request_t *req);
+
+int
+glusterd_handle_cluster_unlock (rpcsvc_request_t *req);
+
+int
+glusterd_handle_stage_op (rpcsvc_request_t *req);
+
+int
+glusterd_handle_commit_op (rpcsvc_request_t *req);
+
+int
+glusterd_handle_cli_probe (rpcsvc_request_t *req);
+
+int
+glusterd_handle_create_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_defrag_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req);
+
+int
+glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
+                              int32_t op_errno, char *op_errstr, char *hostname,
+                              int port, dict_t *dict);
+
+int
+glusterd_op_commit_send_resp (rpcsvc_request_t *req,
+                              int32_t op, int32_t status, char *op_errstr,
+                              dict_t *rsp_dict);
+
+int
+glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int port);
+
+int
+glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
+                        uuid_t uuid, dict_t *dict);
+
+int
+glusterd_handle_cli_deprobe (rpcsvc_request_t *req);
+
+int
+glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req);
+
+int32_t
+glusterd_list_friends (rpcsvc_request_t *req, dict_t *dict, int32_t flags);
+
+int
+glusterd_handle_cli_list_friends (rpcsvc_request_t *req);
+
+int
+glusterd_handle_cli_start_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_friend_update (rpcsvc_request_t *req);
+
+int
+glusterd_handle_cli_stop_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_cli_delete_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_cli_get_volume (rpcsvc_request_t *req);
+
+int32_t
+glusterd_get_volumes (rpcsvc_request_t *req, dict_t *dict, int32_t flags);
+
+int
+glusterd_handle_add_brick (rpcsvc_request_t *req);
+
+int
+glusterd_handle_replace_brick (rpcsvc_request_t *req);
+
+int
+glusterd_handle_remove_brick (rpcsvc_request_t *req);
+
+int
+glusterd_handle_log_rotate (rpcsvc_request_t *req);
+
+int
+glusterd_handle_sync_volume (rpcsvc_request_t *req);
+
+int32_t
+glusterd_log_filename (rpcsvc_request_t *req, dict_t *dict);
+
+int32_t
+glusterd_log_rotate (rpcsvc_request_t *req, dict_t *dict);
+
+int32_t
+glusterd_remove_brick (rpcsvc_request_t *req, dict_t *dict);
+
+int32_t
+glusterd_set_volume (rpcsvc_request_t *req, dict_t *dict);
+
+int32_t
+glusterd_reset_volume (rpcsvc_request_t *req, dict_t *dict);
+
+int32_t
+glusterd_gsync_set (rpcsvc_request_t *req, dict_t *dict);
+
+int32_t
+glusterd_quota (rpcsvc_request_t *req, dict_t *dict);
+
+int
+glusterd_handle_set_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_reset_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_copy_file (rpcsvc_request_t *req);
+
+int
+glusterd_handle_sys_exec (rpcsvc_request_t *req);
+
+int
+glusterd_handle_gsync_set (rpcsvc_request_t *req);
+
+int
+glusterd_handle_quota (rpcsvc_request_t *req);
+
+int
+glusterd_handle_fsm_log (rpcsvc_request_t *req);
+
+int
+glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
+                                int32_t op_errno, char *op_errstr,
+                                char *hostname, dict_t *dict);
+
+int
+glusterd_fetchspec_notify (xlator_t *this);
+
+int
+glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
+                                    dict_t  *volumes, int   count);
+
+int
+glusterd_restart_bricks (glusterd_conf_t *conf);
+
+int32_t
+glusterd_volume_txn (rpcsvc_request_t *req, char *volname, int flags,
+                     glusterd_op_t op);
+
+int
+glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
+                            glusterd_peerctx_t *peerctx);
+
+int
+glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict, char **op_errstr);
+int
+glusterd_handle_cli_profile_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_getwd (rpcsvc_request_t *req);
+
+int32_t
+glusterd_set_volume (rpcsvc_request_t *req, dict_t *dict);
+int
+glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event,
+                          void *data);
+int
+glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data);
+
+int
+glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+                             rpc_clnt_event_t event, void *data);
+
+int
+glusterd_rpc_create (struct rpc_clnt **rpc, dict_t *options,
+                     rpc_clnt_notify_t notify_fn, void *notify_data);
+
+
+/* handler functions */
+int32_t glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                           char *err_str, size_t size);
+
+/* removed other definitions as they have been defined elsewhere in this file*/
+
+int glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req);
+int glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req);
+
+int glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
+                                  size_t len, int cmd, defrag_cbk_fn_t cbk,
+                                  glusterd_op_t op);
+int
+glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
+                               glusterd_conf_t *priv, int cmd);
+
+int glusterd_handle_cli_heal_volume (rpcsvc_request_t *req);
+
+int glusterd_handle_cli_list_volume (rpcsvc_request_t *req);
+
+int
+glusterd_handle_snapshot (rpcsvc_request_t *req);
+
+/* op-sm functions */
+int glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_heal_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr);
+int glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int glusterd_op_stage_copy_file (dict_t *dict, char **op_errstr);
+int glusterd_op_copy_file (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_sys_exec (dict_t *dict, char **op_errstr);
+int glusterd_op_sys_exec (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int glusterd_op_stage_gsync_create (dict_t *dict, char **op_errstr);
+int glusterd_op_gsync_create (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int glusterd_op_stage_quota (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
+                                     dict_t *rsp_dict);
+int glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict);
+int glusterd_op_log_rotate (dict_t *dict);
+int glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_create_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_start_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_stop_volume (dict_t *dict);
+int glusterd_op_delete_volume (dict_t *dict);
+
+int glusterd_op_add_brick (dict_t *dict, char **op_errstr);
+int glusterd_op_remove_brick (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr);
+int glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr);
+
+int glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr);
+int glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int glusterd_op_stage_statedump_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_statedump_volume (dict_t *dict, char **op_errstr);
+
+int glusterd_op_stage_clearlocks_volume (dict_t *dict, char **op_errstr);
+int glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr,
+                                   dict_t *rsp_dict);
+
+/* misc */
+void glusterd_do_replace_brick (void *data);
+int glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
+                                      int force, int *need_migrate);
+int glusterd_op_stop_volume_args_get (dict_t *dict, char** volname, int *flags);
+int glusterd_op_statedump_volume_args_get (dict_t *dict, char **volname,
+                                           char **options, int *option_cnt);
+
+int glusterd_op_gsync_args_get (dict_t *dict, char **op_errstr,
+                                char **master, char **slave, char **host_uuid);
+int glusterd_stop_volume (glusterd_volinfo_t *volinfo);
+
+/* Synctask part */
+int32_t glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
+                                    void *dict);
+int32_t
+glusterd_defrag_event_notify_handle (dict_t *dict);
+
+int32_t
+glusterd_txn_opinfo_dict_init ();
+
+void
+glusterd_txn_opinfo_dict_fini ();
+
+void
+glusterd_txn_opinfo_init ();
+
+/* snapshot */
+glusterd_snap_t*
+glusterd_new_snap_object();
+
+int32_t
+glusterd_list_add_snapvol (glusterd_volinfo_t *origin_vol,
+                           glusterd_volinfo_t *snap_vol);
+
+glusterd_snap_t*
+glusterd_remove_snap_by_id (uuid_t snap_id);
+
+glusterd_snap_t*
+glusterd_remove_snap_by_name (char *snap_name);
+
+glusterd_snap_t*
+glusterd_find_snap_by_name (char *snap_name);
+
+glusterd_snap_t*
+glusterd_find_snap_by_id (uuid_t snap_id);
+
+int
+glusterd_snapshot_prevalidate (dict_t *dict, char **op_errstr,
+                               dict_t *rsp_dict);
+int
+glusterd_snapshot_brickop (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int
+glusterd_snapshot (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int
+glusterd_snapshot_postvalidate (dict_t *dict, int32_t op_ret, char **op_errstr,
+                                dict_t *rsp_dict);
+char *
+glusterd_build_snap_device_path (char *device, char *snapname);
+int32_t
+glusterd_snap_remove (dict_t *rsp_dict, glusterd_snap_t *snap,
+                      gf_boolean_t remove_lvm, gf_boolean_t force);
+int32_t
+glusterd_snapshot_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int32_t
+glusterd_add_missed_snaps_to_list (dict_t *dict, int32_t missed_snap_count);
+
+int32_t
+glusterd_store_missed_snaps_list (char *missed_info, int32_t brick_num,
+                                  char *brick_path, int32_t snap_op,
+                                  int32_t snap_status);
+
+#endif
diff --git a/xlators/mount/fuse/src/Makefile.am b/xlators/mount/fuse/src/Makefile.am
index 9d8d45e4f..653121d18 100644
--- a/xlators/mount/fuse/src/Makefile.am
+++ b/xlators/mount/fuse/src/Makefile.am
@@ -1,14 +1,36 @@
+noinst_HEADERS_linux = $(CONTRIBDIR)/fuse-include/fuse_kernel.h\
+	$(CONTRIBDIR)/fuse-include/mount_util.h\
+	$(CONTRIBDIR)/fuse-lib/mount-gluster-compat.h
+noinst_HEADERS_darwin =	$(CONTRIBDIR)/fuse-include/fuse_kernel_macfuse.h
+noinst_HEADERS_common = $(CONTRIBDIR)/fuse-include/fuse-mount.h\
+	$(CONTRIBDIR)/fuse-include/fuse-misc.h fuse-mem-types.h \
+	fuse-bridge.h
 
-noinst_HEADERS = fuse-extra.h
+if GF_DARWIN_HOST_OS
+    noinst_HEADERS = $(noinst_HEADERS_common) $(noinst_HEADERS_darwin)
+else
+    noinst_HEADERS = $(noinst_HEADERS_common) $(noinst_HEADERS_linux)
+endif
 
 xlator_LTLIBRARIES = fuse.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mount
-fuse_la_SOURCES = fuse-bridge.c fuse-extra.c
-fuse_la_LDFLAGS = -module -avoidversion -shared -nostartfiles $(GF_FUSE_LDADD) 
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) -Wall \
-	-I$(top_srcdir)/libglusterfs/src $(GF_CFLAGS) -DFUSE_USE_VERSION=26
+if GF_DARWIN_HOST_OS
+    mount_source=$(CONTRIBDIR)/macfuse/mount_darwin.c
+else
+    mount_source=$(CONTRIBDIR)/fuse-lib/mount.c $(CONTRIBDIR)/fuse-lib/mount-common.c
+endif
 
+fuse_la_SOURCES = fuse-helpers.c fuse-resolve.c fuse-bridge.c \
+	$(CONTRIBDIR)/fuse-lib/misc.c $(mount_source)
 
-CLEANFILES = 
+fuse_la_LDFLAGS = -module -avoid-version
+fuse_la_LIBADD = @GF_FUSE_LDADD@
 
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/fuse-include \
+	-I$(CONTRIBDIR)/fuse-lib $(GF_FUSE_CFLAGS)
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 43d3af76a..6a5587c2d 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -1,2288 +1,3513 @@
 /*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
+#include <sys/wait.h>
+#include "fuse-bridge.h"
+#include "mount-gluster-compat.h"
+#include "glusterfs.h"
+#include "glusterfs-acl.h"
+
+#ifdef __NetBSD__
+#undef open /* in perfuse.h, pulled from mount-gluster-compat.h */
+#endif
+
+static int gf_fuse_conn_err_log;
+static int gf_fuse_xattr_enotsup_log;
+
+void fini (xlator_t *this_xl);
+
+static void fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino);
+
 /*
- * TODO:
- * Need to free_state() when fuse_reply_err() + return.
- * Check loc->path for "" after fuse_loc_fill in all fops
- * (now being done in getattr, lookup) or better - make 
- * fuse_loc_fill() and inode_path() return success/failure.
+ * Send an invalidate notification up to fuse to purge the file from local
+ * page cache.
  */
+static int32_t
+fuse_invalidate(xlator_t *this, inode_t *inode)
+{
+        fuse_private_t *priv = this->private;
+        uint64_t nodeid;
+
+        /*
+         * NOTE: We only invalidate at the moment if fopen_keep_cache is
+         * enabled because otherwise this is a departure from default
+         * behavior. Specifically, the performance/write-behind xlator
+         * causes unconditional invalidations on write requests.
+         */
+        if (!priv->fopen_keep_cache)
+                return 0;
 
-#include <stdint.h>
-#include <signal.h>
-#include <pthread.h>
+        nodeid = inode_to_fuse_nodeid(inode);
+        gf_log(this->name, GF_LOG_DEBUG, "Invalidate inode id %lu.", nodeid);
+        fuse_log_eh (this, "Sending invalidate inode id: %lu gfid: %s", nodeid,
+                     uuid_utoa (inode->gfid));
+        fuse_invalidate_inode(this, nodeid);
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif /* _CONFIG_H */
+        return 0;
+}
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "xlator.h"
-#include "glusterfs.h"
-#include "defaults.h"
-#include "common-utils.h"
+static int32_t
+fuse_forget_cbk (xlator_t *this, inode_t *inode)
+{
+        //Nothing to free in inode ctx, hence return.
+        return 0;
+}
+
+void
+fuse_inode_set_need_lookup (inode_t *inode, xlator_t *this)
+{
+	uint64_t          need_lookup = 1;
 
-#include <fuse/fuse_lowlevel.h>
+	if (!inode || !this)
+		return;
 
-#include "fuse-extra.h"
-#include "list.h"
-#include "dict.h"
+	inode_ctx_set (inode, this, &need_lookup);
 
-#include "compat.h"
-#include "compat-errno.h"
+	return;
+}
 
-/* TODO: when supporting posix acl, remove this definition */
-#define DISABLE_POSIX_ACL
 
-#define ZR_MOUNTPOINT_OPT   "mountpoint"
-#define ZR_DIRECT_IO_OPT    "direct-io-mode"
+gf_boolean_t
+fuse_inode_needs_lookup (inode_t *inode, xlator_t *this)
+{
+	uint64_t          need_lookup = 0;
+	gf_boolean_t      ret = _gf_false;
 
-#define BIG_FUSE_CHANNEL_SIZE 1048576
+	if (!inode || !this)
+		return ret;
 
-struct fuse_private {
-        int                  fd;
-        struct fuse         *fuse;
-        struct fuse_session *se;
-        struct fuse_chan    *ch;
-        char                *volfile;
-        size_t               volfile_size;
-        char                *mount_point;
-        data_t              *buf;
-        pthread_t            fuse_thread;
-        char                 fuse_thread_started;
-        uint32_t             direct_io_mode;
-        double               entry_timeout;
-        double               attribute_timeout;
+	inode_ctx_get (inode, this, &need_lookup);
+	if (need_lookup)
+		ret = _gf_true;
+	need_lookup = 0;
+	inode_ctx_set (inode, this, &need_lookup);
 
-};
-typedef struct fuse_private fuse_private_t;
-
-#define _FI_TO_FD(fi) ((fd_t *)((long)fi->fh))
-
-#define FI_TO_FD(fi) ((_FI_TO_FD (fi))?(fd_ref (_FI_TO_FD(fi))):((fd_t *) 0))
-
-#define FUSE_FOP(state, ret, op_num, fop, args ...)                     \
-        do {                                                            \
-                call_frame_t *frame = get_call_frame_for_req (state, 1); \
-                xlator_t *xl = frame->this->children ?                  \
-                        frame->this->children->xlator : NULL;           \
-                dict_t *refs = frame->root->req_refs;                   \
-                frame->root->state = state;                             \
-                frame->root->op   = op_num;				\
-                STACK_WIND (frame, ret, xl, xl->fops->fop, args);       \
-                dict_unref (refs);                                      \
-        } while (0)
-
-
-typedef struct {
-        void          *pool;
-        xlator_t      *this;
-        inode_table_t *itable;
-        loc_t          loc;
-        loc_t          loc2;
-        fuse_req_t     req;
-        int32_t        flags;
-        off_t          off;
-        size_t         size;
-        unsigned long  nlookup;
-        fd_t          *fd;
-        dict_t        *dict;
-        char          *name;
-        char           is_revalidate;
-} fuse_state_t;
-
-int fuse_chan_receive (struct fuse_chan *ch,
-                       char *buf,
-                       int32_t size);
+	return ret;
+}
 
 
-static void
-free_state (fuse_state_t *state)
+fuse_fd_ctx_t *
+__fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
 {
-        loc_wipe (&state->loc);
+        uint64_t       val    = 0;
+        int32_t        ret    = 0;
+        fuse_fd_ctx_t *fd_ctx = NULL;
 
-        loc_wipe (&state->loc2);
+        ret = __fd_ctx_get (fd, this, &val);
 
-        if (state->dict) {
-                dict_unref (state->dict);
-                state->dict = (void *)0xaaaaeeee;
-        }
-        if (state->name) {
-                FREE (state->name);
-                state->name = NULL;
-        }
-        if (state->fd) {
-                fd_unref (state->fd);
-                state->fd = (void *)0xfdfdfdfd;
+        fd_ctx = (fuse_fd_ctx_t *)(unsigned long) val;
+
+        if (fd_ctx == NULL) {
+                fd_ctx = GF_CALLOC (1, sizeof (*fd_ctx),
+                                    gf_fuse_mt_fd_ctx_t);
+                if (!fd_ctx) {
+                    goto out;
+                }
+                ret = __fd_ctx_set (fd, this,
+                                    (uint64_t)(unsigned long)fd_ctx);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+                                "fd-ctx-set failed");
+                        GF_FREE (fd_ctx);
+                        fd_ctx = NULL;
+                }
         }
-#ifdef DEBUG
-        memset (state, 0x90, sizeof (*state));
-#endif
-        FREE (state);
-        state = NULL;
+out:
+        return fd_ctx;
 }
 
-
-fuse_state_t *
-state_from_req (fuse_req_t req)
+fuse_fd_ctx_t *
+fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
 {
-        fuse_state_t *state;
-        xlator_t *this = NULL;
+        fuse_fd_ctx_t *fd_ctx = NULL;
 
-        this = fuse_req_userdata (req);
+        if ((fd == NULL) || (this == NULL)) {
+                goto out;
+        }
 
-        state = (void *)calloc (1, sizeof (*state));
-        ERR_ABORT (state);
-        state->pool = this->ctx->pool;
-        state->itable = this->itable;
-        state->req = req;
-        state->this = this;
+        LOCK (&fd->lock);
+        {
+                fd_ctx = __fuse_fd_ctx_check_n_create (this, fd);
+        }
+        UNLOCK (&fd->lock);
 
-        return state;
+out:
+        return fd_ctx;
 }
 
-static pid_t
-get_pid_from_req (fuse_req_t req)
+fuse_fd_ctx_t *
+fuse_fd_ctx_get (xlator_t *this, fd_t *fd)
 {
-        const struct fuse_ctx *ctx = NULL;
-        ctx = fuse_req_ctx(req);
-        return ctx->pid;
+        fuse_fd_ctx_t *fdctx = NULL;
+        uint64_t       value = 0;
+        int            ret   = 0;
+
+        ret = fd_ctx_get (fd, this, &value);
+        if (ret < 0) {
+                goto out;
+        }
+
+        fdctx = (fuse_fd_ctx_t *) (unsigned long)value;
+
+out:
+        return fdctx;
 }
 
-static call_frame_t *
-get_call_frame_for_req (fuse_state_t *state, char d)
+/*
+ * iov_out should contain a fuse_out_header at zeroth position.
+ * The error value of this header is sent to kernel.
+ */
+static int
+send_fuse_iov (xlator_t *this, fuse_in_header_t *finh, struct iovec *iov_out,
+               int count)
 {
-        call_pool_t *pool = state->pool;
-        fuse_req_t req = state->req;
-        const struct fuse_ctx *ctx = NULL;
-	call_frame_t *frame = NULL;
-        xlator_t *this = NULL;
         fuse_private_t *priv = NULL;
+        struct fuse_out_header *fouh = NULL;
+        int res, i;
 
+        if (!this || !finh || !iov_out) {
+                gf_log ("send_fuse_iov", GF_LOG_ERROR,"Invalid arguments");
+                return EINVAL;
+        }
+        priv = this->private;
 
-	if (req) {
-		this = fuse_req_userdata (req);
-	} else {
-		this = state->this;
-	}
-	priv = this->private;
+        fouh = iov_out[0].iov_base;
+        iov_out[0].iov_len = sizeof (*fouh);
+        fouh->len = 0;
+        for (i = 0; i < count; i++)
+                fouh->len += iov_out[i].iov_len;
+        fouh->unique = finh->unique;
 
-	frame = create_frame (this, pool);
+        res = writev (priv->fd, iov_out, count);
 
-        if (req) {
-                ctx = fuse_req_ctx(req);
+        if (res == -1)
+                return errno;
+        if (res != fouh->len)
+                return EINVAL;
 
-                frame->root->uid    = ctx->uid;
-                frame->root->gid    = ctx->gid;
-                frame->root->pid    = ctx->pid;
-                frame->root->unique = req_callid (req);
-        }
+        if (priv->fuse_dump_fd != -1) {
+                char w = 'W';
 
-        if (d) {
-                frame->root->req_refs = dict_ref (get_new_dict ());
-                dict_set (frame->root->req_refs, NULL, priv->buf);
+                pthread_mutex_lock (&priv->fuse_dump_mutex);
+                res = write (priv->fuse_dump_fd, &w, 1);
+                if (res != -1)
+                        res = writev (priv->fuse_dump_fd, iov_out, count);
+                pthread_mutex_unlock (&priv->fuse_dump_mutex);
+
+                if (res == -1)
+                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                "failed to dump fuse message (W): %s",
+                                strerror (errno));
         }
 
-        frame->root->type = GF_OP_TYPE_FOP_REQUEST;
+        return 0;
+}
+
+static int
+send_fuse_data (xlator_t *this, fuse_in_header_t *finh, void *data, size_t size)
+{
+        struct fuse_out_header fouh = {0, };
+        struct iovec iov_out[2];
+
+        fouh.error = 0;
+        iov_out[0].iov_base = &fouh;
+        iov_out[1].iov_base = data;
+        iov_out[1].iov_len = size;
 
-        return frame;
+        return send_fuse_iov (this, finh, iov_out, 2);
 }
 
+#define send_fuse_obj(this, finh, obj) \
+        send_fuse_data (this, finh, obj, sizeof (*(obj)))
 
-GF_MUST_CHECK static int32_t
-fuse_loc_fill (loc_t *loc,
-               fuse_state_t *state,
-               ino_t ino,
-               ino_t par,
-               const char *name)
+
+static void
+fuse_invalidate_entry (xlator_t *this, uint64_t fuse_ino)
 {
-        inode_t *inode = NULL, *parent = NULL;
-	int32_t ret = -1;
-	char *path = NULL;
+        struct fuse_out_header             *fouh   = NULL;
+        struct fuse_notify_inval_entry_out *fnieo  = NULL;
+        fuse_private_t                     *priv   = NULL;
+        dentry_t                           *dentry = NULL;
+        inode_t                            *inode  = NULL;
+        size_t                              nlen   = 0;
+        int                                 rv     = 0;
+        char inval_buf[INVAL_BUF_SIZE]             = {0,};
+
+        fouh  = (struct fuse_out_header *)inval_buf;
+        fnieo = (struct fuse_notify_inval_entry_out *)(fouh + 1);
+
+        priv = this->private;
+        if (priv->revchan_out == -1)
+                return;
+
+        fouh->unique = 0;
+        fouh->error = FUSE_NOTIFY_INVAL_ENTRY;
+
+        inode = fuse_ino_to_inode (fuse_ino, this);
 
-        /* resistance against multiple invocation of loc_fill not to get
-           reference leaks via inode_search() */
+        list_for_each_entry (dentry, &inode->dentry_list, inode_list) {
+                nlen = strlen (dentry->name);
+                fouh->len = sizeof (*fouh) + sizeof (*fnieo) + nlen + 1;
+                fnieo->parent = inode_to_fuse_nodeid (dentry->parent);
 
-        inode = loc->inode;
-	
-        if (!inode) {
-                if (ino)
-                        inode = inode_search (state->itable, ino, NULL);
-                if (par && name)
-                        inode = inode_search (state->itable, par, name);
+                fnieo->namelen = nlen;
+                strcpy (inval_buf + sizeof (*fouh) + sizeof (*fnieo), dentry->name);
 
-                loc->inode = inode;
-                if (inode)
-                        loc->ino = inode->ino;
+                rv = write (priv->revchan_out, inval_buf, fouh->len);
+                if (rv != fouh->len) {
+                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                "kernel notification daemon defunct");
+
+                        close (priv->fd);
+                        break;
+                }
+
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE entry: "
+                        "%"PRIu64"/%s", fnieo->parent, dentry->name);
+
+                if (dentry->parent) {
+                        fuse_log_eh (this, "Invalidated entry %s (parent: %s)",
+                                     dentry->name,
+                                     uuid_utoa (dentry->parent->gfid));
+                } else {
+                        fuse_log_eh (this, "Invalidated entry %s(nodeid: %ld)",
+                                     dentry->name, fnieo->parent);
+                }
         }
 
-        parent = loc->parent;
-        if (!parent) {
-                if (inode)
-                        parent = inode_parent (inode, par, name);
-                else
-                        parent = inode_search (state->itable, par, NULL);
-                loc->parent = parent;
-        }
-  
-        if (name && parent) {
-		ret = inode_path (parent, name, &path);
-		if (ret <= 0) {
-			gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-				"inode_path failed for %"PRId64"/%s",
-				parent->ino, name);
-			goto fail;
-		} else {
-			loc->path = path;
-		}
-        } else 	if (inode) {
-		ret = inode_path (inode, NULL, &path);
-		if (ret <= 0) {
-			gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-				"inode_path failed for %"PRId64,
-				inode->ino);
-			goto fail;
-		} else {
-			loc->path = path;
-		}
-	}
-	if (loc->path) {
-		loc->name = strrchr (loc->path, '/');
-		if (loc->name)
-			loc->name++;
-		else loc->name = "";
-	}
-	
-	if ((ino != 1) &&
-	    (parent == NULL)) {
-		gf_log ("fuse-bridge", GF_LOG_ERROR,
-			"failed to search parent for %"PRId64"/%s (%"PRId64")",
-			(ino_t)par, name, (ino_t)ino);
-		ret = -1;
-		goto fail;
-	}
-	ret = 0;
-fail:
-	return ret;
+        if (inode)
+                inode_unref (inode);
 }
 
+/*
+ * Send an inval inode notification to fuse. This causes an invalidation of the
+ * entire page cache mapping on the inode.
+ */
+static void
+fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino)
+{
+        struct fuse_out_header *fouh = NULL;
+        struct fuse_notify_inval_inode_out *fniio = NULL;
+        fuse_private_t *priv = NULL;
+        int rv = 0;
+        char inval_buf[INVAL_BUF_SIZE] = {0};
+        inode_t    *inode = NULL;
+
+        fouh = (struct fuse_out_header *) inval_buf;
+        fniio = (struct fuse_notify_inval_inode_out *) (fouh + 1);
 
-static int
-need_fresh_lookup (int32_t op_ret, int32_t op_errno, 
-		   loc_t *loc, struct stat *buf)
-{
-        if (op_ret == -1) {
-		gf_log ("fuse-bridge",
-			(op_errno == ENOENT)? GF_LOG_DEBUG: GF_LOG_WARNING,
-			"revalidate of %s failed (%s)",
-			loc->path, strerror (op_errno));
-                return 1;
-	}
+        priv = this->private;
+
+        if (priv->revchan_out < 0)
+                return;
 
-        if (loc->inode->ino != buf->st_ino) {
-		gf_log ("fuse-bridge", GF_LOG_WARNING,
-			"inode num of %s changed %"PRId64" -> %"PRId64,
-			loc->path, loc->inode->ino, buf->st_ino);
-                return 1;
+        fouh->unique = 0;
+        fouh->error = FUSE_NOTIFY_INVAL_INODE;
+        fouh->len = sizeof(struct fuse_out_header) +
+                sizeof(struct fuse_notify_inval_inode_out);
+
+        /* inval the entire mapping until we learn how to be more granular */
+        fniio->ino = fuse_ino;
+        fniio->off = 0;
+        fniio->len = -1;
+
+        inode = fuse_ino_to_inode (fuse_ino, this);
+
+        rv = write(priv->revchan_out, inval_buf, fouh->len);
+        if (rv != fouh->len) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR, "kernel notification "
+                        "daemon defunct");
+                close(priv->fd);
         }
 
-	if ((loc->inode->st_mode & S_IFMT) ^ (buf->st_mode & S_IFMT)) {
-		gf_log ("fuse-bridge", GF_LOG_WARNING,
-			"inode mode of %s changed 0%o -> 0%o",
-			loc->path, loc->inode->st_mode, buf->st_mode);
-		return 1;
-	}
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE inode: %lu", fuse_ino);
 
-        return 0;
+        if (inode) {
+                fuse_log_eh (this, "Invalidated inode %lu (gfid: %s)",
+                             fuse_ino, uuid_utoa (inode->gfid));
+        } else {
+                fuse_log_eh (this, "Invalidated inode %lu ", fuse_ino);
+        }
+
+        if (inode)
+                inode_unref (inode);
 }
 
+int
+send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error)
+{
+        struct fuse_out_header fouh = {0, };
+        struct iovec iov_out;
+        inode_t  *inode = NULL;
 
-static int
-fuse_lookup_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 inode_t *inode,
-                 struct stat *stat,
-                 dict_t *dict);
+        fouh.error = -error;
+        iov_out.iov_base = &fouh;
+
+        inode = fuse_ino_to_inode (finh->nodeid, this);
+
+        // filter out ENOENT
+        if (error != ENOENT) {
+                if (inode) {
+                        fuse_log_eh (this,"Sending %s for operation %d on "
+                                     "inode %s", strerror (error), finh->opcode,
+                                     uuid_utoa (inode->gfid));
+                } else {
+                        fuse_log_eh (this, "Sending %s for operation %d on "
+                                     "inode %ld", strerror (error),
+                                     finh->opcode, finh->nodeid);
+                }
+        }
+
+        if (inode)
+                inode_unref (inode);
+
+        return send_fuse_iov (this, finh, &iov_out, 1);
+}
 
 static int
-fuse_entry_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                inode_t *inode,
-                struct stat *buf)
+fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno,
+                inode_t *inode, struct iatt *buf, dict_t *xdata)
 {
-        fuse_state_t *state;
-        fuse_req_t req;
-        struct fuse_entry_param e = {0, };
-        fuse_private_t *priv = this->private;
+        fuse_state_t          *state        = NULL;
+        fuse_in_header_t      *finh         = NULL;
+        struct fuse_entry_out  feo          = {0, };
+        fuse_private_t        *priv         = NULL;
+        inode_t               *linked_inode = NULL;
 
+        priv = this->private;
         state = frame->root->state;
-        req = state->req;
+        finh = state->finh;
 
-        if (!op_ret && state->loc.ino == 1) {
-                buf->st_ino = 1;
+        if (op_ret == 0) {
+                if (__is_root_gfid (state->loc.inode->gfid))
+                        buf->ia_ino = 1;
+                if (uuid_is_null (buf->ia_gfid)) {
+                        /* With a NULL gfid inode linking is
+                           not possible. Let's not pretend this
+                           call was a "success".
+                        */
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "Received NULL gfid for %s. Forcing EIO",
+                                state->loc.path);
+                        op_ret = -1;
+                        op_errno = EIO;
+                }
         }
 
-        if (state->is_revalidate == 1
-	    && need_fresh_lookup (op_ret, op_errno, &state->loc, buf)) {
-                inode_unref (state->loc.inode);
-                state->loc.inode = inode_new (state->itable);
-                state->is_revalidate = 2;
+        /* log into the event-history after the null uuid check is done, since
+         * the op_ret and op_errno are being changed if the gfid is NULL.
+         */
+        fuse_log_eh (this, "op_ret: %d op_errno: %d "
+                     "%"PRIu64": %s() %s => %s", op_ret, op_errno,
+                     frame->root->unique, gf_fop_list[frame->root->op],
+                     state->loc.path, (op_ret == 0)?
+                     uuid_utoa(buf->ia_gfid):uuid_utoa(state->loc.gfid));
 
-                STACK_WIND (frame, fuse_lookup_cbk,
-                            FIRST_CHILD (this),
-			    FIRST_CHILD (this)->fops->lookup,
-                            &state->loc, state->dict);
+        if (op_ret == 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s() %s => %"PRIu64,
+                        frame->root->unique, gf_fop_list[frame->root->op],
+                        state->loc.path, buf->ia_ino);
 
-                return 0;
-        }
+                buf->ia_blksize = this->ctx->page_size;
+                gf_fuse_stat2attr (buf, &feo.attr, priv->enable_ino32);
 
-        if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": %s() %s => %"PRId64" (%"PRId64")",
-			frame->root->unique, gf_fop_list[frame->root->op],
-			state->loc.path, buf->st_ino, state->loc.ino);
+                if (!buf->ia_ino) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "%"PRIu64": %s() %s returning inode 0",
+                                frame->root->unique,
+                                gf_fop_list[frame->root->op], state->loc.path);
+                }
 
-		inode_link (inode, state->loc.parent, state->loc.name, buf);
+                linked_inode = inode_link (inode, state->loc.parent,
+                                           state->loc.name, buf);
 
-		inode_lookup (inode);
+                if (linked_inode != inode) {
+                }
 
-                /* TODO: make these timeouts configurable (via meta?) */
-                e.ino = inode->ino;
+                inode_lookup (linked_inode);
 
-#ifdef GF_DARWIN_HOST_OS
-                e.generation = 0;
-#else
-                e.generation = buf->st_ctime;
-#endif
+                feo.nodeid = inode_to_fuse_nodeid (linked_inode);
 
-                e.entry_timeout = priv->entry_timeout;
-                e.attr_timeout  = priv->attribute_timeout;
-                e.attr = *buf;
-                e.attr.st_blksize = BIG_FUSE_CHANNEL_SIZE; 
-  
-		if (!e.ino || !buf->st_ino) {
-			gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-				"%"PRId64": %s() %s returning inode 0",
-				frame->root->unique,
-				gf_fop_list[frame->root->op], state->loc.path);
-		}
+                inode_unref (linked_inode);
 
-                if (state->loc.parent)
-                        fuse_reply_entry (req, &e);
-                else
-                        fuse_reply_attr (req, buf, priv->attribute_timeout);
+                feo.entry_valid =
+                        calc_timeout_sec (priv->entry_timeout);
+                feo.entry_valid_nsec =
+                        calc_timeout_nsec (priv->entry_timeout);
+                feo.attr_valid =
+                        calc_timeout_sec (priv->attribute_timeout);
+                feo.attr_valid_nsec =
+                        calc_timeout_nsec (priv->attribute_timeout);
+
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+                priv->proto_minor >= 9 ?
+                send_fuse_obj (this, finh, &feo) :
+                send_fuse_data (this, finh, &feo,
+                                FUSE_COMPAT_ENTRY_OUT_SIZE);
+#else
+                send_fuse_obj (this, finh, &feo);
+#endif
         } else {
                 gf_log ("glusterfs-fuse",
-			(op_errno == ENOENT ? GF_LOG_DEBUG : GF_LOG_ERROR),
-                        "%"PRId64": %s() %s => -1 (%s)", frame->root->unique,
+                        (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_WARNING),
+                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
                         gf_fop_list[frame->root->op], state->loc.path,
-			strerror (op_errno));
-                fuse_reply_err (req, op_errno);
+                        strerror (op_errno));
+
+		if ((op_errno == ENOENT) && (priv->negative_timeout != 0)) {
+			feo.entry_valid =
+				calc_timeout_sec (priv->negative_timeout);
+			feo.entry_valid_nsec =
+				calc_timeout_nsec (priv->negative_timeout);
+			send_fuse_obj (this, finh, &feo);
+		} else {
+			send_fuse_err (this, state->finh, op_errno);
+                }
         }
 
-        free_state (state);
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
         return 0;
 }
 
-
 static int
-fuse_lookup_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 inode_t *inode,
-                 struct stat *stat,
-                 dict_t *dict)
-{
-        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, stat);
+fuse_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, buf,
+                        xdata);
         return 0;
 }
 
-
-static void
-fuse_lookup (fuse_req_t req,
-             fuse_ino_t par,
-             const char *name)
+static int
+fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 inode_t *inode, struct iatt *stat, dict_t *dict,
+                 struct iatt *postparent)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
-	
-        state = state_from_req (req);
-
-        ret = fuse_loc_fill (&state->loc, state, 0, par, name);
-
-	if (ret < 0) {
-		gf_log ("glusterfs-fuse", GF_LOG_ERROR, 
-			"%"PRId64": LOOKUP %"PRId64"/%s (fuse_loc_fill() failed)", 
-			req_callid (req), (ino_t)par, name);
-		free_state (state);
-		fuse_reply_err (req, EINVAL);
-		return;
-	}
+        fuse_state_t            *state = NULL;
+        call_frame_t            *prev = NULL;
+        inode_table_t           *itable = NULL;
 
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": LOOKUP %s", req_callid (req),
+        state = frame->root->state;
+        prev  = cookie;
+
+        if (op_ret == -1 && state->is_revalidate == 1) {
+                itable = state->itable;
+		/*
+		 * A stale mapping might exist for a dentry/inode that has been
+		 * removed from another client.
+		 */
+		if (op_errno == ENOENT)
+			inode_unlink(state->loc.inode, state->loc.parent,
+				     state->loc.name);
+                inode_unref (state->loc.inode);
+                state->loc.inode = inode_new (itable);
+                state->is_revalidate = 2;
+                if (uuid_is_null (state->gfid))
+                        uuid_generate (state->gfid);
+                fuse_gfid_set (state);
+
+                STACK_WIND (frame, fuse_lookup_cbk,
+                            prev->this, prev->this->fops->lookup,
+                            &state->loc, state->xdata);
+                return 0;
+        }
+
+        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, stat,
+                        dict);
+        return 0;
+}
+
+void
+fuse_lookup_resume (fuse_state_t *state)
+{
+        if (!state->loc.parent && !state->loc.inode) {
+                gf_log ("fuse", GF_LOG_ERROR, "failed to resolve path %s",
                         state->loc.path);
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
+                return;
+        }
 
-                state->loc.inode = inode_new (state->itable);
-                /* to differntiate in entry_cbk what kind of call it is */
-                state->is_revalidate = -1;
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": LOOKUP %s(%"PRId64")", req_callid (req),
-                        state->loc.path, state->loc.inode->ino);
+        /* parent was resolved, entry could not, may be a missing gfid?
+         * Hence try to do a regular lookup
+         */
+        if ((state->resolve.op_ret == -1)
+            && (state->resolve.op_errno == ENODATA)) {
+                state->resolve.op_ret = 0;
+        }
+
+        if (state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": LOOKUP %s(%s)", state->finh->unique,
+                        state->loc.path, uuid_utoa (state->loc.inode->gfid));
                 state->is_revalidate = 1;
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": LOOKUP %s", state->finh->unique,
+                        state->loc.path);
+                state->loc.inode = inode_new (state->loc.parent->table);
+                if (uuid_is_null (state->gfid))
+                        uuid_generate (state->gfid);
+                fuse_gfid_set (state);
         }
-	
-	state->dict = dict_new();
 
         FUSE_FOP (state, fuse_lookup_cbk, GF_FOP_LOOKUP,
-                  lookup, &state->loc, state->dict);
+                  lookup, &state->loc, state->xdata);
 }
 
+static void
+fuse_lookup (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        char           *name     = msg;
+        fuse_state_t   *state    = NULL;
+
+        GET_STATE (this, finh, state);
+
+        (void) fuse_resolve_entry_init (state, &state->resolve,
+                                        finh->nodeid, name);
+
+        fuse_resolve_and_resume (state, fuse_lookup_resume);
+
+        return;
+}
+
+static inline void
+do_forget(xlator_t *this, uint64_t unique, uint64_t nodeid, uint64_t nlookup)
+{
+	inode_t *fuse_inode = fuse_ino_to_inode(nodeid, this);
+
+	fuse_log_eh(this, "%"PRIu64": FORGET %"PRIu64"/%"PRIu64" gfid: (%s)",
+		    unique, nodeid, nlookup, uuid_utoa(fuse_inode->gfid));
+
+	inode_forget(fuse_inode, nlookup);
+	inode_unref(fuse_inode);
+}
 
 static void
-fuse_forget (fuse_req_t req,
-             fuse_ino_t ino,
-             unsigned long nlookup)
+fuse_forget (xlator_t *this, fuse_in_header_t *finh, void *msg)
+
 {
-        inode_t *fuse_inode;
-        fuse_state_t *state;
+        struct fuse_forget_in *ffi        = msg;
 
-        if (ino == 1) {
-                fuse_reply_none (req);
+        if (finh->nodeid == 1) {
+                GF_FREE (finh);
                 return;
         }
 
-        state = state_from_req (req);
-        fuse_inode = inode_search (state->itable, ino, NULL);
-	if (fuse_inode) {
-		gf_log ("glusterfs-fuse", GF_LOG_DEBUG, 
-			"got forget on inode (%lu)", ino);
-		inode_forget (fuse_inode, nlookup);
-		inode_unref (fuse_inode);
-	} else {
-		gf_log ("glusterfs-fuse", GF_LOG_ERROR, 
-			"got forget, but inode (%lu) not found", ino);
-	}
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": FORGET %"PRIu64"/%"PRIu64,
+                finh->unique, finh->nodeid, ffi->nlookup);
+
+	do_forget(this, finh->unique, finh->nodeid, ffi->nlookup);
 
-        free_state (state);
-        fuse_reply_none (req);
+        GF_FREE (finh);
 }
 
+static void
+fuse_batch_forget(xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+	struct fuse_batch_forget_in *fbfi = msg;
+	struct fuse_forget_one *ffo = (struct fuse_forget_one *) (fbfi + 1);
+	int i;
+
+	gf_log("glusterfs-fuse", GF_LOG_TRACE,
+		"%"PRIu64": BATCH_FORGET %"PRIu64"/%"PRIu32,
+		finh->unique, finh->nodeid, fbfi->count);
+
+	for (i = 0; i < fbfi->count; i++) {
+                if (ffo[i].nodeid == 1)
+                        continue;
+		do_forget(this, finh->unique, ffo[i].nodeid, ffo[i].nlookup);
+        }
+
+	GF_FREE(finh);
+}
 
 static int
-fuse_attr_cbk (call_frame_t *frame,
-               void *cookie,
-               xlator_t *this,
-               int32_t op_ret,
-               int32_t op_errno,
-               struct stat *buf)
+fuse_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        fuse_state_t *state;
-        fuse_req_t req;
-        fuse_private_t *priv = this->private;
+        fuse_state_t     *state;
+        fuse_in_header_t *finh;
+        fuse_private_t   *priv = NULL;
+        struct fuse_attr_out fao;
 
+        priv  = this->private;
         state = frame->root->state;
-        req = state->req;
+        finh  = state->finh;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
         if (op_ret == 0) {
-                gf_log ("glusterfs-fuse",
-			(buf->st_ino ? GF_LOG_DEBUG : GF_LOG_ERROR),
-                        "%"PRId64": %s() %s => %"PRId64, frame->root->unique, 
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
                         gf_fop_list[frame->root->op],
-			state->loc.path ? state->loc.path : "ERR",
-                        buf->st_ino);
+                        state->loc.path ? state->loc.path : "ERR",
+                        prebuf->ia_ino);
+
+                postbuf->ia_blksize = this->ctx->page_size;
+                gf_fuse_stat2attr (postbuf, &fao.attr, priv->enable_ino32);
 
-                /* TODO: make these timeouts configurable via meta */
-                /* TODO: what if the inode number has changed by now */ 
-                buf->st_blksize = BIG_FUSE_CHANNEL_SIZE;
+                fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
+                fao.attr_valid_nsec =
+                  calc_timeout_nsec (priv->attribute_timeout);
 
-                fuse_reply_attr (req, buf, priv->attribute_timeout);
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+                priv->proto_minor >= 9 ?
+                send_fuse_obj (this, finh, &fao) :
+                send_fuse_data (this, finh, &fao,
+                                FUSE_COMPAT_ATTR_OUT_SIZE);
+#else
+                send_fuse_obj (this, finh, &fao);
+#endif
         } else {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": %s() %s => -1 (%s)", frame->root->unique, 
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
                         gf_fop_list[frame->root->op],
-			state->loc.path ? state->loc.path : "ERR", 
+                        state->loc.path ? state->loc.path : "ERR",
                         strerror (op_errno));
 
-                fuse_reply_err (req, op_errno);
+                send_fuse_err (this, finh, op_errno);
         }
-        
-        free_state (state);
+
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
+
         return 0;
 }
 
-
-static void
-fuse_getattr (fuse_req_t req,
-              fuse_ino_t ino,
-              struct fuse_file_info *fi)
+static int
+fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        fuse_state_t *state;
-        fd_t         *fd = NULL;
-	int32_t       ret = -1;
-
-        state = state_from_req (req);
-
-        if (ino == 1) {
-                ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-		if (ret < 0) {
-			gf_log ("glusterfs-fuse", GF_LOG_ERROR, 
-				"%"PRId64": GETATTR %"PRId64" (fuse_loc_fill() failed)",
-				req_callid(req), (ino_t)ino);
-			fuse_reply_err (req, EINVAL);
-			free_state (state);
-			return;
-		}
+        fuse_state_t     *state;
+        fuse_in_header_t *finh;
+        fuse_private_t   *priv = NULL;
+        struct fuse_attr_out fao;
 
-                if (state->loc.inode)
-                        state->is_revalidate = 1;
-                else
-                        state->is_revalidate = -1;
+        priv  = this->private;
+        state = frame->root->state;
+        finh  = state->finh;
 
-		state->dict = dict_new();
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() %s => "
+                   "gfid: %s", op_ret, op_errno, frame->root->unique,
+                   gf_fop_list[frame->root->op], state->loc.path,
+                   state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
+        if (op_ret == 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
+                        gf_fop_list[frame->root->op],
+                        state->loc.path ? state->loc.path : "ERR",
+                        buf->ia_ino);
 
-                FUSE_FOP (state, fuse_lookup_cbk, GF_FOP_LOOKUP,
-                          lookup, &state->loc, state->dict);
-                return;
+                buf->ia_blksize = this->ctx->page_size;
+                gf_fuse_stat2attr (buf, &fao.attr, priv->enable_ino32);
+
+                fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
+                fao.attr_valid_nsec =
+                  calc_timeout_nsec (priv->attribute_timeout);
+
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+                priv->proto_minor >= 9 ?
+                send_fuse_obj (this, finh, &fao) :
+                send_fuse_data (this, finh, &fao,
+                                FUSE_COMPAT_ATTR_OUT_SIZE);
+#else
+                send_fuse_obj (this, finh, &fao);
+#endif
+        } else {
+                GF_LOG_OCCASIONALLY ( gf_fuse_conn_err_log, "glusterfs-fuse",
+                                      GF_LOG_WARNING,
+                                      "%"PRIu64": %s() %s => -1 (%s)",
+                                      frame->root->unique,
+                                      gf_fop_list[frame->root->op],
+                                      state->loc.path ? state->loc.path : "ERR",
+                                      strerror (op_errno));
+
+                send_fuse_err (this, finh, op_errno);
         }
 
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
+        free_fuse_state (state);
+        STACK_DESTROY (frame->root);
+
+        return 0;
+}
 
+static int
+fuse_root_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      inode_t *inode, struct iatt *stat, dict_t *dict,
+                      struct iatt *postparent)
+{
+        fuse_attr_cbk (frame, cookie, this, op_ret, op_errno, stat, dict);
+
+        return 0;
+}
+
+void
+fuse_getattr_resume (fuse_state_t *state)
+{
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": GETATTR %"PRId64" (%s) (fuse_loc_fill() returned NULL inode)", 
-                        req_callid (req), (int64_t)ino, state->loc.path);
-                fuse_reply_err (req, EINVAL);
+                        "%"PRIu64": GETATTR %"PRIu64" (%s) resolution failed",
+                        state->finh->unique, state->finh->nodeid,
+                        uuid_utoa (state->resolve.gfid));
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
-        
-        fd = fd_lookup (state->loc.inode, get_pid_from_req (req));
-        state->fd = fd;
-        if (!fd || S_ISDIR (state->loc.inode->st_mode)) {
-		/* this is the @ret of fuse_loc_fill, checked here
-		   to permit fstat() to happen even when fuse_loc_fill fails
-		*/
-		if (ret < 0) {
-			gf_log ("glusterfs-fuse", GF_LOG_ERROR, 
-				"%"PRId64": GETATTR %"PRId64" (fuse_loc_fill() failed)",
-				req_callid(req), (ino_t)ino);
-			fuse_reply_err (req, EINVAL);
-			free_state (state);
-			return;
-		}
 
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": GETATTR %"PRId64" (%s)",
-                        req_callid (req), (int64_t)ino, state->loc.path);
+        if (!IA_ISDIR (state->loc.inode->ia_type)) {
+                state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+        }
+
+        if (!state->fd) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": GETATTR %"PRIu64" (%s)",
+                        state->finh->unique, state->finh->nodeid,
+                        state->loc.path);
 
-    
                 FUSE_FOP (state, fuse_attr_cbk, GF_FOP_STAT,
-                          stat, &state->loc);
+                          stat, &state->loc, state->xdata);
         } else {
 
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": FGETATTR %"PRId64" (%s/%p)",
-                        req_callid (req), (int64_t)ino, state->loc.path, fd);
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": FGETATTR %"PRIu64" (%s/%p)",
+                        state->finh->unique, state->finh->nodeid,
+                        state->loc.path, state->fd);
 
-                FUSE_FOP (state,fuse_attr_cbk, GF_FOP_FSTAT,
-                          fstat, fd);
+                FUSE_FOP (state, fuse_attr_cbk, GF_FOP_FSTAT,
+                          fstat, state->fd, state->xdata);
         }
 }
 
+static void
+fuse_getattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        fuse_state_t *state;
+        int32_t       ret = -1;
+
+        GET_STATE (this, finh, state);
+
+        if (finh->nodeid == 1) {
+                state->gfid[15] = 1;
+
+                ret = fuse_loc_fill (&state->loc, state, finh->nodeid, 0, NULL);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "%"PRIu64": GETATTR on / (fuse_loc_fill() failed)",
+                                finh->unique);
+                        send_fuse_err (this, finh, ENOENT);
+                        free_fuse_state (state);
+                        return;
+                }
+
+                fuse_gfid_set (state);
+
+                FUSE_FOP (state, fuse_root_lookup_cbk, GF_FOP_LOOKUP,
+                          lookup, &state->loc, state->xdata);
+                return;
+        }
+
+        fuse_resolve_inode_init (state, &state->resolve, state->finh->nodeid);
+
+        fuse_resolve_and_resume (state, fuse_getattr_resume);
+}
+
+static int32_t
+fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
+{
+        int32_t        ret    = 0;
+        fuse_fd_ctx_t *fdctx  = NULL, *tmp_fdctx = NULL;
+        fd_t          *tmp_fd = NULL;
+
+        GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", this, out, ret,
+                                        -EINVAL);
+        GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", fd, out, ret,
+                                        -EINVAL);
+        GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", foo, out, ret,
+                                        -EINVAL);
+
+        fdctx = fuse_fd_ctx_get (this, fd);
+        if (!fdctx) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        tmp_fd = fd_lookup (fd->inode, 0);
+        if (tmp_fd) {
+                tmp_fdctx = fuse_fd_ctx_get (this, tmp_fd);
+                if (tmp_fdctx) {
+                        foo->open_flags &= ~FOPEN_DIRECT_IO;
+                        foo->open_flags |= (tmp_fdctx->open_flags
+                                            & FOPEN_DIRECT_IO);
+                }
+        }
+
+        fdctx->open_flags |= (foo->open_flags & FOPEN_DIRECT_IO);
+
+        if (tmp_fd != NULL) {
+                fd_unref (tmp_fd);
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
 
 static int
-fuse_fd_cbk (call_frame_t *frame,
-             void *cookie,
-             xlator_t *this,
-             int32_t op_ret,
-             int32_t op_errno,
-             fd_t *fd)
+fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        fuse_state_t *state;
-        fuse_req_t req;
-        fuse_private_t *priv = this->private;
+        fuse_state_t         *state  = NULL;
+        fuse_in_header_t     *finh   = NULL;
+        fuse_private_t       *priv   = NULL;
+        int32_t               ret    = 0;
+        struct fuse_open_out  foo    = {0, };
 
+        priv = this->private;
         state = frame->root->state;
-        req = state->req;
+        finh = state->finh;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
         if (op_ret >= 0) {
-                struct fuse_file_info fi = {0, };
-                
-                fi.fh = (unsigned long) fd;
-                fi.flags = state->flags;
-
-                if (!S_ISDIR (fd->inode->st_mode)) {
-                        if ((fi.flags & 3) && priv->direct_io_mode)
-                                fi.direct_io = 1;
+                foo.fh = (uintptr_t) fd;
+                foo.open_flags = 0;
+
+                if (!IA_ISDIR (fd->inode->ia_type)) {
+                        if (((priv->direct_io_mode == 2)
+                             && ((state->flags & O_ACCMODE) != O_RDONLY))
+                            || (priv->direct_io_mode == 1))
+                                foo.open_flags |= FOPEN_DIRECT_IO;
+#ifdef GF_DARWIN_HOST_OS
+                        /* In Linux: by default, buffer cache
+                         * is purged upon open, setting
+                         * FOPEN_KEEP_CACHE implies no-purge
+                         *
+                         * In MacFUSE: by default, buffer cache
+                         * is left intact upon open, setting
+                         * FOPEN_PURGE_UBC implies purge
+                         *
+                         * [[Interesting...]]
+                         */
+                        if (!priv->fopen_keep_cache)
+                                foo.open_flags |= FOPEN_PURGE_UBC;
+#else
+                        /*
+                         * If fopen-keep-cache is enabled, we set the associated
+                         * flag here such that files are not invalidated on open.
+                         * File invalidations occur either in fuse or explicitly
+                         * when the cache is set invalid on the inode.
+                         */
+                        if (priv->fopen_keep_cache)
+                                foo.open_flags |= FOPEN_KEEP_CACHE;
+#endif
                 }
 
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": %s() %s => %p", frame->root->unique,
-			gf_fop_list[frame->root->op], state->loc.path, fd);
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s() %s => %p", frame->root->unique,
+                        gf_fop_list[frame->root->op], state->loc.path, fd);
 
-		fd_ref (fd);
-                if (fuse_reply_open (req, &fi) == -ENOENT) {
+                ret = fuse_fd_inherit_directio (this, fd, &foo);
+                if (ret < 0) {
+                        op_errno = -ret;
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-				"open() got EINTR");
-			fd_unref (fd);
-				goto out;
+                                "cannot inherit direct-io values for fd "
+                                "(ptr:%p inode-gfid:%s) from fds already "
+                                "opened", fd, uuid_utoa (fd->inode->gfid));
+                        goto err;
                 }
-		
-		fd_bind (fd);
+
+                if (send_fuse_obj (this, finh, &foo) == ENOENT) {
+                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+                                "open(%s) got EINTR", state->loc.path);
+                        gf_fd_put (priv->fdtable, state->fd_no);
+                        goto out;
+                }
+
+                fd_bind (fd);
         } else {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": %s() %s => -1 (%s)", frame->root->unique,
+        err:
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
                         gf_fop_list[frame->root->op], state->loc.path,
-			strerror (op_errno));
+                        strerror (op_errno));
 
-                fuse_reply_err (req, op_errno);
+                send_fuse_err (this, finh, op_errno);
+                gf_fd_put (priv->fdtable, state->fd_no);
         }
 out:
-        free_state (state);
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
         return 0;
 }
 
+static void
+fuse_do_truncate (fuse_state_t *state, size_t size)
+{
+        if (state->fd) {
+                FUSE_FOP (state, fuse_truncate_cbk, GF_FOP_FTRUNCATE,
+                          ftruncate, state->fd, size, state->xdata);
+        } else {
+                FUSE_FOP (state, fuse_truncate_cbk, GF_FOP_TRUNCATE,
+                          truncate, &state->loc, size, state->xdata);
+        }
 
+        return;
+}
 
-static void
-do_chmod (fuse_req_t req,
-          fuse_ino_t ino,
-          struct stat *attr,
-          struct fuse_file_info *fi)
+static int
+fuse_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
-        fuse_state_t *state = state_from_req (req);
-        fd_t *fd = NULL;
-	int32_t ret = -1;
+        fuse_state_t     *state;
+        fuse_in_header_t *finh;
+        fuse_private_t   *priv = NULL;
+        struct fuse_attr_out fao;
 
-        if (fi) {
-                fd = FI_TO_FD (fi);
-                state->fd = fd;
-        }
+        int op_done = 0;
 
-        if (fd) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": FCHMOD %p", req_callid (req), fd);
+        priv  = this->private;
+        state = frame->root->state;
+        finh  = state->finh;
 
-                FUSE_FOP (state, fuse_attr_cbk, GF_FOP_FCHMOD,
-                          fchmod, fd, attr->st_mode);
-        } else {
-                ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
+        fuse_log_eh(this, "op_ret: %d, op_errno: %d, %"PRIu64", %s() %s => "
+                    "gfid: %s", op_ret, op_errno, frame->root->unique,
+                    gf_fop_list[frame->root->op], state->loc.path,
+                    state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
 
-                if ((state->loc.inode == NULL) ||
-		    (ret < 0)) {
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "%"PRId64": CHMOD %"PRId64" (%s) (fuse_loc_fill() failed)", 
-                                req_callid (req), (int64_t)ino,
-				state->loc.path);
-                        fuse_reply_err (req, EINVAL);
-			free_state (state);
-                        return;
-                }
+        if (op_ret == 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
+                        gf_fop_list[frame->root->op],
+                        state->loc.path ? state->loc.path : "ERR",
+                        statpost->ia_ino);
 
+                statpost->ia_blksize = this->ctx->page_size;
+                gf_fuse_stat2attr (statpost, &fao.attr, priv->enable_ino32);
 
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": CHMOD %s", req_callid (req),
-                        state->loc.path);
+                fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
+                fao.attr_valid_nsec =
+                        calc_timeout_nsec (priv->attribute_timeout);
 
-                FUSE_FOP (state, fuse_attr_cbk, GF_FOP_CHMOD,
-                          chmod, &state->loc, attr->st_mode);
+                if (state->truncate_needed) {
+                        fuse_do_truncate (state, state->size);
+                } else {
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+                        priv->proto_minor >= 9 ?
+                        send_fuse_obj (this, finh, &fao) :
+                        send_fuse_data (this, finh, &fao,
+                                        FUSE_COMPAT_ATTR_OUT_SIZE);
+#else
+                        send_fuse_obj (this, finh, &fao);
+#endif
+                        op_done = 1;
+                }
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
+                        gf_fop_list[frame->root->op],
+                        state->loc.path ? state->loc.path : "ERR",
+                        strerror (op_errno));
+
+                send_fuse_err (this, finh, op_errno);
+                op_done = 1;
         }
-}
 
+        if (op_done) {
+                free_fuse_state (state);
+        }
 
-static void
-do_chown (fuse_req_t req,
-          fuse_ino_t ino,
-          struct stat *attr,
-          int valid,
-          struct fuse_file_info *fi)
+        STACK_DESTROY (frame->root);
+
+        return 0;
+}
+
+static int32_t
+fattr_to_gf_set_attr (int32_t valid)
 {
-        fuse_state_t *state;
-        fd_t *fd = NULL;
-	int32_t ret = -1;
-        uid_t uid = (valid & FUSE_SET_ATTR_UID) ? attr->st_uid : (uid_t) -1;
-        gid_t gid = (valid & FUSE_SET_ATTR_GID) ? attr->st_gid : (gid_t) -1;
+        int32_t gf_valid = 0;
 
-        state = state_from_req (req);
+        if (valid & FATTR_MODE)
+                gf_valid |= GF_SET_ATTR_MODE;
 
-        if (fi) {
-                fd = FI_TO_FD (fi);
-                state->fd = fd;
-        }
+        if (valid & FATTR_UID)
+                gf_valid |= GF_SET_ATTR_UID;
 
-        if (fd) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": FCHOWN %p", req_callid (req), fd);
+        if (valid & FATTR_GID)
+                gf_valid |= GF_SET_ATTR_GID;
 
-                FUSE_FOP (state, fuse_attr_cbk, GF_FOP_FCHOWN,
-                          fchown, fd, uid, gid);
-        } else {
-                ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-                if ((state->loc.inode == NULL) ||
-		    (ret < 0)) {
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "%"PRId64": CHOWN %"PRId64" (%s) (fuse_loc_fill() failed)", 
-                                req_callid (req), (int64_t)ino,
-				state->loc.path);
-                        fuse_reply_err (req, EINVAL);
-			free_state (state);
-                        return;
-                }
+        if (valid & FATTR_ATIME)
+                gf_valid |= GF_SET_ATTR_ATIME;
 
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": CHOWN %s", req_callid (req),
-                        state->loc.path);
+        if (valid & FATTR_MTIME)
+                gf_valid |= GF_SET_ATTR_MTIME;
 
-                FUSE_FOP (state, fuse_attr_cbk, GF_FOP_CHOWN,
-                          chown, &state->loc, uid, gid);
-        }
+        if (valid & FATTR_SIZE)
+                gf_valid |= GF_SET_ATTR_SIZE;
+
+        return gf_valid;
 }
 
+#define FATTR_MASK   (FATTR_SIZE                        \
+                      | FATTR_UID | FATTR_GID           \
+                      | FATTR_ATIME | FATTR_MTIME       \
+                      | FATTR_MODE)
 
-static void 
-do_truncate (fuse_req_t req,
-             fuse_ino_t ino,
-             struct stat *attr,
-             struct fuse_file_info *fi)
+void
+fuse_setattr_resume (fuse_state_t *state)
 {
-        fuse_state_t *state;
-        fd_t *fd = NULL;
-	int32_t ret = -1;
-
-        state = state_from_req (req);
-        
-        if (fi) {
-                fd = FI_TO_FD (fi);
-                state->fd = fd;
+        if (!state->fd && !state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "%"PRIu64": SETATTR %"PRIu64" (%s) resolution failed",
+                        state->finh->unique, state->finh->nodeid,
+                        uuid_utoa (state->resolve.gfid));
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
+                return;
         }
-        if (fd) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": FTRUNCATE %p/%"PRId64, req_callid (req),
-                        fd, attr->st_size);
 
-                FUSE_FOP (state, fuse_attr_cbk, GF_FOP_FTRUNCATE,
-                          ftruncate, fd, attr->st_size);
-        } else {
-                ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-                if ((state->loc.inode == NULL) || 
-		    (ret < 0)) {
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "%"PRId64": TRUNCATE %s/%"PRId64" (fuse_loc_fill() failed)", 
-                                req_callid (req), state->loc.path,
-				attr->st_size);
-                        fuse_reply_err (req, EINVAL);
-			free_state (state);
-                        return;
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": SETATTR (%"PRIu64")%s", state->finh->unique,
+                state->finh->nodeid, state->loc.path);
+
+#ifdef GF_TEST_FFOP
+        /* this is for calls like 'fchmod()' */
+        if (!state->fd)
+                state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+#endif /* GF_TEST_FFOP */
+
+        if ((state->valid & (FATTR_MASK)) != FATTR_SIZE) {
+                if (state->fd &&
+                    !((state->valid & FATTR_ATIME) ||
+                      (state->valid & FATTR_MTIME))) {
+                        /*
+                            there is no "futimes" call, so don't send
+                            fsetattr if ATIME or MTIME is set
+                         */
+
+                        FUSE_FOP (state, fuse_setattr_cbk, GF_FOP_FSETATTR,
+                                  fsetattr, state->fd, &state->attr,
+                                  fattr_to_gf_set_attr (state->valid),
+                                  state->xdata);
+                } else {
+                        FUSE_FOP (state, fuse_setattr_cbk, GF_FOP_SETATTR,
+                                  setattr, &state->loc, &state->attr,
+                                  fattr_to_gf_set_attr (state->valid),
+                                  state->xdata);
                 }
-
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": TRUNCATE %s/%"PRId64"(%lu)",
-			req_callid (req),
-                        state->loc.path, attr->st_size, ino);
-
-                FUSE_FOP (state, fuse_attr_cbk, GF_FOP_TRUNCATE,
-                          truncate, &state->loc, attr->st_size);
+        } else {
+                fuse_do_truncate (state, state->size);
         }
 
-        return;
 }
 
-
-static void 
-do_utimes (fuse_req_t req,
-           fuse_ino_t ino,
-           struct stat *attr)
+static void
+fuse_setattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
+        struct fuse_setattr_in *fsi = msg;
 
-        struct timespec tv[2];
-	int32_t ret = -1;
+        fuse_private_t  *priv = NULL;
+        fuse_state_t *state = NULL;
 
-	tv[0].tv_sec = attr->st_atime;
-	tv[0].tv_nsec = ST_ATIM_NSEC(attr);
-        tv[1].tv_sec = attr->st_mtime;
-        tv[1].tv_nsec = ST_ATIM_NSEC(attr);
+        GET_STATE (this, finh, state);
 
-        state = state_from_req (req);
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": UTIMENS %s (fuse_loc_fill() failed)", 
-                        req_callid (req), state->loc.path);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
-                return;
+        if (fsi->valid & FATTR_FH &&
+            !(fsi->valid & (FATTR_ATIME|FATTR_MTIME))) {
+                /* We need no loc if kernel sent us an fd and
+                 * we are not fiddling with times */
+                state->fd = FH_TO_FD (fsi->fh);
+                fuse_resolve_fd_init (state, &state->resolve, state->fd);
+        } else {
+                fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
         }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": UTIMENS (%lu)%s", req_callid (req),
-                ino, state->loc.path);
-
-        FUSE_FOP (state, fuse_attr_cbk, GF_FOP_UTIMENS,
-                  utimens, &state->loc, tv);
-}
+        /*
+         * This is just stub code demonstrating how to retrieve
+         * lock_owner in setattr, according to the FUSE proto.
+         * We do not make use of ATM. Its purpose is supporting
+         * mandatory locking, but getting that right is further
+         * down the road. Cf.
+         *
+         * http://thread.gmane.org/gmane.comp.file-systems.fuse.devel/
+         * 4962/focus=4982
+         *
+         * http://git.kernel.org/?p=linux/kernel/git/torvalds/
+         * linux-2.6.git;a=commit;h=v2.6.23-5896-gf333211
+         */
+        priv = this->private;
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+        if (priv->proto_minor >= 9 && fsi->valid & FATTR_LOCKOWNER)
+                state->lk_owner = fsi->lock_owner;
+#endif
 
+        state->valid = fsi->valid;
 
-static void
-fuse_setattr (fuse_req_t req,
-              fuse_ino_t ino,
-              struct stat *attr,
-              int valid,
-              struct fuse_file_info *fi)
-{
+        if ((fsi->valid & (FATTR_MASK)) != FATTR_SIZE) {
+                if (fsi->valid & FATTR_SIZE) {
+                        state->size            = fsi->size;
+                        state->truncate_needed = _gf_true;
+                }
 
-        if (valid & FUSE_SET_ATTR_MODE)
-                do_chmod (req, ino, attr, fi);
-        else if (valid & (FUSE_SET_ATTR_UID | FUSE_SET_ATTR_GID))
-                do_chown (req, ino, attr, valid, fi);
-        else if (valid & FUSE_SET_ATTR_SIZE)
-                do_truncate (req, ino, attr, fi);
-        else if (valid & (FUSE_SET_ATTR_ATIME | FUSE_SET_ATTR_MTIME))
-                do_utimes (req, ino, attr);
-        else 
-                fuse_getattr (req, ino, fi);
-}
+                state->attr.ia_size  = fsi->size;
+                state->attr.ia_atime = fsi->atime;
+                state->attr.ia_mtime = fsi->mtime;
+                state->attr.ia_atime_nsec = fsi->atimensec;
+                state->attr.ia_mtime_nsec = fsi->mtimensec;
 
+                state->attr.ia_prot = ia_prot_from_st_mode (fsi->mode);
+                state->attr.ia_uid  = fsi->uid;
+                state->attr.ia_gid  = fsi->gid;
+        } else {
+                state->size = fsi->size;
+        }
 
-static int gf_fuse_xattr_enotsup_log;
+        fuse_resolve_and_resume (state, fuse_setattr_resume);
+}
 
 static int
-fuse_err_cbk (call_frame_t *frame,
-              void *cookie,
-              xlator_t *this,
-              int32_t op_ret,
-              int32_t op_errno)
+fuse_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         fuse_state_t *state = frame->root->state;
-        fuse_req_t req = state->req;
+        fuse_in_header_t *finh = state->finh;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
         if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": %s() %s => 0", frame->root->unique,
-			gf_fop_list[frame->root->op], 
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s() %s => 0", frame->root->unique,
+                        gf_fop_list[frame->root->op],
                         state->loc.path ? state->loc.path : "ERR");
 
-                fuse_reply_err (req, 0);
+                send_fuse_err (this, finh, 0);
         } else {
-                if (frame->root->op == GF_FOP_SETXATTR) {
-			op_ret = gf_compat_setxattr (state->dict);
-			if (op_ret == 0)
-				op_errno = 0;
-			if (op_errno == ENOTSUP) {
-				gf_fuse_xattr_enotsup_log++;
-				if (!(gf_fuse_xattr_enotsup_log % GF_UNIVERSAL_ANSWER))
-					gf_log ("glusterfs-fuse", GF_LOG_CRITICAL,
-						"[ ERROR ] Extended attribute not supported by the backend storage");
-			}
-                } else {
-			if ((frame->root->op == GF_FOP_REMOVEXATTR)
-			    && (op_errno == ENOATTR)) {
-				goto nolog;
-			}
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "%"PRId64": %s() %s => -1 (%s)",
-				frame->root->unique,
-				gf_fop_list[frame->root->op],
-				state->loc.path ? state->loc.path : "ERR",
-                                strerror (op_errno));
-                }
-	nolog:
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": %s() %s => -1 (%s)",
+                        frame->root->unique,
+                        gf_fop_list[frame->root->op],
+                        state->loc.path ? state->loc.path : "ERR",
+                        strerror (op_errno));
 
-                fuse_reply_err (req, op_errno);
+                send_fuse_err (this, finh, op_errno);
         }
 
-        free_state (state);
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
+static int
+fuse_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+}
 
+static int
+fuse_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        if (op_ret == -1 && op_errno == ENOTSUP)
+                GF_LOG_OCCASIONALLY (gf_fuse_xattr_enotsup_log,
+                                     "glusterfs-fuse", GF_LOG_CRITICAL,
+                                     "extended attribute not supported "
+                                     "by the backend storage");
+
+        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+}
 
 static int
-fuse_unlink_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno)
+fuse_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
-        fuse_req_t req = state->req;
+        fuse_state_t     *state = NULL;
+        fuse_in_header_t *finh = NULL;
 
-        if (op_ret == 0)
-                inode_unlink (state->loc.inode, state->loc.parent,
-			      state->loc.name);
+        state = frame->root->state;
+        finh = state->finh;
+
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() %s => "
+                     "gfid: %s", op_ret, op_errno, frame->root->unique,
+                     gf_fop_list[frame->root->op], state->loc.path,
+                     state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
 
         if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": %s() %s => 0", frame->root->unique,
+                inode_unlink (state->loc.inode, state->loc.parent,
+                              state->loc.name);
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s() %s => 0", frame->root->unique,
                         gf_fop_list[frame->root->op], state->loc.path);
 
-                fuse_reply_err (req, 0);
+                send_fuse_err (this, finh, 0);
         } else {
-                gf_log ("glusterfs-fuse", 
-                        (op_errno != ENOTEMPTY ? GF_LOG_ERROR : GF_LOG_DEBUG),
-                        "%"PRId64": %s() %s => -1 (%s)", frame->root->unique,
+                gf_log ("glusterfs-fuse",
+                        op_errno == ENOTEMPTY ? GF_LOG_DEBUG : GF_LOG_WARNING,
+                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
                         gf_fop_list[frame->root->op], state->loc.path,
-			strerror (op_errno));
+                        strerror (op_errno));
 
-                fuse_reply_err (req, op_errno);
+                send_fuse_err (this, finh, op_errno);
         }
 
-        free_state (state);
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
+void
+fuse_access_resume (fuse_state_t *state)
+{
+        if (!state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "%"PRIu64": ACCESS %"PRIu64" (%s) resolution failed",
+                        state->finh->unique, state->finh->nodeid,
+                        uuid_utoa (state->resolve.gfid));
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
+                return;
+        }
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64" ACCESS %s/%"PRIu64" mask=%d",
+                state->finh->unique, state->loc.path,
+                state->finh->nodeid, state->mask);
+
+        FUSE_FOP (state, fuse_err_cbk, GF_FOP_ACCESS, access,
+                  &state->loc, state->mask, state->xdata);
+}
 
 static void
-fuse_access (fuse_req_t req,
-             fuse_ino_t ino,
-             int mask)
+fuse_access (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
+        struct fuse_access_in *fai = msg;
+        fuse_state_t *state = NULL;
 
-        state = state_from_req (req);
+        GET_STATE (this, finh, state);
 
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-        if ((state->loc.inode == NULL) || 
-	    (ret < 0)) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": ACCESS %"PRId64" (%s) (fuse_loc_fill() failed)", 
-                        req_callid (req), (int64_t)ino, state->loc.path);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
-                return;
-        }
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64" ACCESS %s/%lu mask=%d", req_callid (req),
-                state->loc.path, ino, mask);
+        state->mask = fai->mask;
 
-        FUSE_FOP (state, fuse_err_cbk,
-                  GF_FOP_ACCESS, access,
-		  &state->loc, mask);
+        fuse_resolve_and_resume (state, fuse_access_resume);
 
         return;
 }
 
-
-
 static int
-fuse_readlink_cbk (call_frame_t *frame,
-                   void *cookie,
-                   xlator_t *this,
-                   int32_t op_ret,
-                   int32_t op_errno,
-                   const char *linkname)
+fuse_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, const char *linkname,
+                   struct iatt *buf, dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
-        fuse_req_t req = state->req;
+        fuse_state_t     *state = NULL;
+        fuse_in_header_t *finh = NULL;
+
+        state = frame->root->state;
+        finh = state->finh;
+
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d %"PRIu64": %s() => %s"
+                     " linkname: %s, gfid: %s", op_ret, op_errno,
+                     frame->root->unique, gf_fop_list[frame->root->op],
+                     state->loc.gfid, linkname,
+                     uuid_utoa (state->loc.gfid));
 
         if (op_ret > 0) {
                 ((char *)linkname)[op_ret] = '\0';
 
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": %s => %s", frame->root->unique,
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s => %s", frame->root->unique,
                         state->loc.path, linkname);
 
-                fuse_reply_readlink(req, linkname);
+                send_fuse_data (this, finh, (void *)linkname, op_ret + 1);
         } else {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": %s => -1 (%s)", frame->root->unique,
-                        state->loc.path, strerror(op_errno));
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": %s => -1 (%s)", frame->root->unique,
+                        state->loc.path, strerror (op_errno));
 
-                fuse_reply_err(req, op_errno);
+                send_fuse_err (this, finh, op_errno);
         }
 
-        free_state (state);
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
-
-static void
-fuse_readlink (fuse_req_t req,
-               fuse_ino_t ino)
+void
+fuse_readlink_resume (fuse_state_t *state)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
-
-        state = state_from_req (req);
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
+        if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64" READLINK %s/%"PRId64" (fuse_loc_fill() returned NULL inode)", 
-                        req_callid (req), state->loc.path,
-			state->loc.inode->ino);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+                        "READLINK %"PRIu64" (%s) resolution failed",
+                        state->finh->unique, uuid_utoa (state->resolve.gfid));
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
-  
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64" READLINK %s/%"PRId64, req_callid (req),
-                state->loc.path, state->loc.inode->ino);
 
-        FUSE_FOP (state, fuse_readlink_cbk, GF_FOP_READLINK,
-		  readlink, &state->loc, 4096);
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64" READLINK %s/%s", state->finh->unique,
+                state->loc.path, uuid_utoa (state->loc.inode->gfid));
 
-        return;
+        FUSE_FOP (state, fuse_readlink_cbk, GF_FOP_READLINK,
+                  readlink, &state->loc, 4096, state->xdata);
 }
 
-
 static void
-fuse_mknod (fuse_req_t req,
-            fuse_ino_t par,
-            const char *name,
-            mode_t mode,
-            dev_t rdev)
+fuse_readlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
-
-        state = state_from_req (req);
-        ret = fuse_loc_fill (&state->loc, state, 0, par, name);
-	if (ret < 0) {
-		gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64" MKNOD %s (fuse_loc_fill() failed)", 
-                        req_callid (req), state->loc.path);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
-                return;
-	}
+        fuse_state_t *state = NULL;
 
-        state->loc.inode = inode_new (state->itable);
+        GET_STATE (this, finh, state);
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": MKNOD %s", req_callid (req),
-                state->loc.path);
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
 
-        FUSE_FOP (state, fuse_entry_cbk, GF_FOP_MKNOD,
-		  mknod, &state->loc, mode, rdev);
+        fuse_resolve_and_resume (state, fuse_readlink_resume);
 
         return;
 }
 
-
-static void 
-fuse_mkdir (fuse_req_t req,
-            fuse_ino_t par,
-            const char *name,
-            mode_t mode)
+void
+fuse_mknod_resume (fuse_state_t *state)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
-
-        state = state_from_req (req);
-        ret = fuse_loc_fill (&state->loc, state, 0, par, name);
-	if (ret < 0) {
-		gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64" MKDIR %s (fuse_loc_fill() failed)", 
-                        req_callid (req), state->loc.path);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+        if (!state->loc.parent) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "MKNOD %"PRIu64"/%s (%s/%s) resolution failed",
+                        state->finh->nodeid, state->resolve.bname,
+                        uuid_utoa (state->resolve.gfid), state->resolve.bname);
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
-	}
+        }
+
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
+        if (state->loc.inode) {
+                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
+                inode_unref (state->loc.inode);
+                state->loc.inode = NULL;
+        }
 
-        state->loc.inode = inode_new (state->itable);
+        state->loc.inode = inode_new (state->loc.parent->table);
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": MKDIR %s", req_callid (req),
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": MKNOD %s", state->finh->unique,
                 state->loc.path);
 
-        FUSE_FOP (state, fuse_entry_cbk, GF_FOP_MKDIR,
-		  mkdir, &state->loc, mode);
+        FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_MKNOD,
+                  mknod, &state->loc, state->mode, state->rdev, state->umask,
+                  state->xdata);
+}
+
+static void
+fuse_mknod (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        struct fuse_mknod_in *fmi = msg;
+        char         *name = (char *)(fmi + 1);
+
+        fuse_state_t   *state = NULL;
+        fuse_private_t *priv = NULL;
+        int32_t         ret = -1;
+
+        priv = this->private;
+#if FUSE_KERNEL_MINOR_VERSION >= 12
+        if (priv->proto_minor < 12)
+                name = (char *)msg + FUSE_COMPAT_MKNOD_IN_SIZE;
+#endif
+
+        GET_STATE (this, finh, state);
+
+        uuid_generate (state->gfid);
+
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+
+        state->mode = fmi->mode;
+        state->rdev = fmi->rdev;
+
+        priv = this->private;
+#if FUSE_KERNEL_MINOR_VERSION >=12
+        FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKNOD");
+#endif
+
+        fuse_resolve_and_resume (state, fuse_mknod_resume);
 
         return;
 }
 
+void
+fuse_mkdir_resume (fuse_state_t *state)
+{
+        if (!state->loc.parent) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "MKDIR %"PRIu64" (%s/%s) resolution failed",
+                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
+                        state->resolve.bname);
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
+                return;
+        }
+
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
+        if (state->loc.inode) {
+                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
+                inode_unref (state->loc.inode);
+                state->loc.inode = NULL;
+        }
+
+        state->loc.inode = inode_new (state->loc.parent->table);
 
-static void 
-fuse_unlink (fuse_req_t req,
-             fuse_ino_t par,
-             const char *name)
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": MKDIR %s", state->finh->unique,
+                state->loc.path);
+
+        FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_MKDIR,
+                  mkdir, &state->loc, state->mode, state->umask, state->xdata);
+}
+
+static void
+fuse_mkdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
+        struct fuse_mkdir_in *fmi = msg;
+        char *name = (char *)(fmi + 1);
+        fuse_private_t *priv = NULL;
+
         fuse_state_t *state;
-	int32_t ret = -1;
+        int32_t ret = -1;
+
+        GET_STATE (this, finh, state);
 
-        state = state_from_req (req);
+        uuid_generate (state->gfid);
 
-        ret = fuse_loc_fill (&state->loc, state, 0, par, name);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
 
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
+        state->mode = fmi->mode;
+
+        priv = this->private;
+#if FUSE_KERNEL_MINOR_VERSION >=12
+        FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKDIR");
+#endif
+
+        fuse_resolve_and_resume (state, fuse_mkdir_resume);
+
+        return;
+}
+
+void
+fuse_unlink_resume (fuse_state_t *state)
+{
+        if (!state->loc.parent || !state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": UNLINK %s (fuse_loc_fill() returned NULL inode)",
-                        req_callid (req), state->loc.path);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+                        "UNLINK %"PRIu64" (%s/%s) resolution failed",
+                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
+                        state->resolve.bname);
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": UNLINK %s", req_callid (req),
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": UNLINK %s", state->finh->unique,
                 state->loc.path);
 
         FUSE_FOP (state, fuse_unlink_cbk, GF_FOP_UNLINK,
-		  unlink, &state->loc);
+                  unlink, &state->loc, 0, state->xdata);
+}
+
+static void
+fuse_unlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        char         *name = msg;
+        fuse_state_t *state = NULL;
+
+        GET_STATE (this, finh, state);
+
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+
+        fuse_resolve_and_resume (state, fuse_unlink_resume);
 
         return;
 }
 
-
-static void 
-fuse_rmdir (fuse_req_t req,
-            fuse_ino_t par,
-            const char *name)
+void
+fuse_rmdir_resume (fuse_state_t *state)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
-
-        state = state_from_req (req);
-        ret = fuse_loc_fill (&state->loc, state, 0, par, name);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": RMDIR %s (fuse_loc_fill() failed)",
-                        req_callid (req), state->loc.path);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+        if (!state->loc.parent || !state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "RMDIR %"PRIu64" (%s/%s) resolution failed",
+                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
+                        state->resolve.bname);
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": RMDIR %s", req_callid (req),
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": RMDIR %s", state->finh->unique,
                 state->loc.path);
 
         FUSE_FOP (state, fuse_unlink_cbk, GF_FOP_RMDIR,
-		  rmdir, &state->loc);
+                  rmdir, &state->loc, 0, state->xdata);
+}
+
+static void
+fuse_rmdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        char         *name = msg;
+        fuse_state_t *state = NULL;
+
+        GET_STATE (this, finh, state);
+
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+
+        fuse_resolve_and_resume (state, fuse_rmdir_resume);
 
         return;
 }
 
+void
+fuse_symlink_resume (fuse_state_t *state)
+{
+        if (!state->loc.parent) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "SYMLINK %"PRIu64" (%s/%s) -> %s resolution failed",
+                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
+                        state->resolve.bname, state->name);
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
+                return;
+        }
+
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
+        if (state->loc.inode) {
+                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
+                inode_unref (state->loc.inode);
+                state->loc.inode = NULL;
+        }
+
+        state->loc.inode = inode_new (state->loc.parent->table);
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": SYMLINK %s -> %s", state->finh->unique,
+                state->loc.path, state->name);
+
+        FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_SYMLINK,
+                  symlink, state->name, &state->loc, state->umask, state->xdata);
+}
 
 static void
-fuse_symlink (fuse_req_t req,
-              const char *linkname,
-              fuse_ino_t par,
-              const char *name)
+fuse_symlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
-
-        state = state_from_req (req);
-        ret = fuse_loc_fill (&state->loc, state, 0, par, name);
-	if (ret < 0) {
-		gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64" SYMLINK %s -> %s (fuse_loc_fill() failed)", 
-                        req_callid (req), state->loc.path, linkname);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
-                return;
-	}
+        char         *name = msg;
+        char         *linkname = name + strlen (name) + 1;
+        fuse_state_t *state = NULL;
 
-        state->loc.inode = inode_new (state->itable);
+        GET_STATE (this, finh, state);
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": SYMLINK %s -> %s", req_callid (req),
-                state->loc.path, linkname);
+        uuid_generate (state->gfid);
 
-        FUSE_FOP (state, fuse_entry_cbk, GF_FOP_SYMLINK,
-                  symlink, linkname, &state->loc);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+
+        state->name = gf_strdup (linkname);
+
+        fuse_resolve_and_resume (state, fuse_symlink_resume);
 
         return;
 }
 
-
-int 
-fuse_rename_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct stat *buf)
+int
+fuse_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 struct iatt *preoldparent, struct iatt *postoldparent,
+                 struct iatt *prenewparent, struct iatt *postnewparent,
+                 dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
-        fuse_req_t req = state->req;
+        fuse_state_t     *state = NULL;
+        fuse_in_header_t *finh = NULL;
+
+        state = frame->root->state;
+        finh  = state->finh;
+
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() "
+                     "path: %s parent: %s ==> path: %s parent: %s"
+                     "gfid: %s", op_ret, op_errno, frame->root->unique,
+                     gf_fop_list[frame->root->op], state->loc.path,
+                     state->loc.parent?uuid_utoa (state->loc.parent->gfid):"",
+                     state->loc2.path,
+                     state->loc2.parent?uuid_utoa (state->loc2.parent->gfid):"",
+                     state->loc.inode?uuid_utoa (state->loc.inode->gfid):"");
 
         if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": %s -> %s => 0 (buf->st_ino=%"PRId64" , loc->ino=%"PRId64")", 
-			frame->root->unique, state->loc.path, state->loc2.path,
-			buf->st_ino, state->loc.ino);
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s -> %s => 0 (buf->ia_ino=%"PRIu64")",
+                        frame->root->unique, state->loc.path, state->loc2.path,
+                        buf->ia_ino);
 
                 {
                         /* ugly ugly - to stay blind to situation where
                            rename happens on a new inode
                         */
-                        buf->st_ino = state->loc.ino;
-			buf->st_mode = state->loc.inode->st_mode;
+                        buf->ia_type = state->loc.inode->ia_type;
                 }
-                inode_rename (state->itable,
+                buf->ia_blksize = this->ctx->page_size;
+
+                inode_rename (state->loc.parent->table,
                               state->loc.parent, state->loc.name,
                               state->loc2.parent, state->loc2.name,
                               state->loc.inode, buf);
 
-                fuse_reply_err (req, 0);
+                send_fuse_err (this, finh, 0);
         } else {
-                gf_log ("glusterfs-fuse",
-			(op_errno == ENOTEMPTY ? GF_LOG_DEBUG : GF_LOG_ERROR),
-                        "%"PRId64": %s -> %s => -1 (%s)", frame->root->unique,
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": %s -> %s => -1 (%s)", frame->root->unique,
                         state->loc.path, state->loc2.path,
-			strerror (op_errno));
-                fuse_reply_err (req, op_errno);
+                        strerror (op_errno));
+                send_fuse_err (this, finh, op_errno);
         }
 
-        free_state (state);
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
         return 0;
 }
 
-
-static void
-fuse_rename (fuse_req_t req,
-             fuse_ino_t oldpar,
-             const char *oldname,
-             fuse_ino_t newpar,
-             const char *newname)
+void
+fuse_rename_resume (fuse_state_t *state)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
+        char loc_uuid[64]  = {0,};
+        char loc2_uuid[64] = {0,};
 
-        state = state_from_req (req);
-
-        ret = fuse_loc_fill (&state->loc, state, 0, oldpar, oldname);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
+        if (!state->loc.parent || !state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "for %s %"PRId64": RENAME `%s' -> `%s' (fuse_loc_fill() failed)",
-                        state->loc.path, req_callid (req), state->loc.path,
-                        state->loc2.path);
-    
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+                        "RENAME %"PRIu64" %s/%s -> %s/%s src resolution failed",
+                        state->finh->unique,
+                        uuid_utoa_r (state->resolve.gfid, loc_uuid),
+                        state->resolve.bname,
+                        uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
+                        state->resolve2.bname);
+
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
 
-        ret = fuse_loc_fill (&state->loc2, state, 0, newpar, newname);
-	if (ret < 0) {
-		gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "for %s %"PRId64": RENAME `%s' -> `%s' (fuse_loc_fill() failed)",
-                        state->loc.path, req_callid (req), state->loc.path,
-                        state->loc2.path);
-    
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+        if (!state->loc2.parent) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "RENAME %"PRIu64" %s/%s -> %s/%s dst resolution failed",
+                        state->finh->unique,
+                        uuid_utoa_r (state->resolve.gfid, loc_uuid),
+                        state->resolve.bname,
+                        uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
+                        state->resolve2.bname);
+
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
-       	}
+        }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": RENAME `%s (%"PRId64")' -> `%s (%"PRId64")'",
-                req_callid (req), state->loc.path, state->loc.ino,
-                state->loc2.path, state->loc2.ino);
+        state->resolve.op_ret = 0;
+        state->resolve2.op_ret = 0;
 
-        FUSE_FOP (state, fuse_rename_cbk, GF_FOP_RENAME,
-                  rename, &state->loc, &state->loc2);
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": RENAME `%s (%s)' -> `%s (%s)'",
+                state->finh->unique, state->loc.path, loc_uuid,
+                state->loc2.path, loc2_uuid);
 
-        return;
+        FUSE_FOP (state, fuse_rename_cbk, GF_FOP_RENAME,
+                  rename, &state->loc, &state->loc2, state->xdata);
 }
 
-
 static void
-fuse_link (fuse_req_t req,
-           fuse_ino_t ino,
-           fuse_ino_t par,
-           const char *name)
+fuse_rename (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
+        struct fuse_rename_in  *fri = msg;
+        char *oldname = (char *)(fri + 1);
+        char *newname = oldname + strlen (oldname) + 1;
+        fuse_state_t *state = NULL;
 
-        state = state_from_req (req);
+        GET_STATE (this, finh, state);
 
-        ret = fuse_loc_fill (&state->loc, state, 0, par, name);
-        ret = fuse_loc_fill (&state->loc2, state, ino, 0, NULL);
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, oldname);
 
-        if ((state->loc2.inode == NULL) ||
-	    (ret < 0)) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "fuse_loc_fill() failed for %s %"PRId64": LINK %s %s", 
-                        state->loc2.path, req_callid (req), 
-                        state->loc2.path, state->loc.path);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+        fuse_resolve_entry_init (state, &state->resolve2, fri->newdir, newname);
+
+        fuse_resolve_and_resume (state, fuse_rename_resume);
+
+        return;
+}
+
+void
+fuse_link_resume (fuse_state_t *state)
+{
+        if (!state->loc2.inode || !state->loc.parent) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "fuse_loc_fill() failed %"PRIu64": LINK %s %s",
+                        state->finh->unique, state->loc2.path, state->loc.path);
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
 
+        state->resolve.op_ret = 0;
+        state->resolve2.op_ret = 0;
+
+        if (state->loc.inode) {
+                inode_unref (state->loc.inode);
+                state->loc.inode = NULL;
+        }
         state->loc.inode = inode_ref (state->loc2.inode);
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": LINK() %s (%"PRId64") -> %s (%"PRId64")",
-		req_callid (req), state->loc2.path, state->loc2.ino,
-		state->loc.path, state->loc.ino);
 
-        FUSE_FOP (state, fuse_entry_cbk, GF_FOP_LINK,
-                  link, &state->loc2, &state->loc);
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": LINK() %s -> %s",
+                state->finh->unique, state->loc2.path,
+                state->loc.path);
 
-        return;
+        FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_LINK,
+                  link, &state->loc2, &state->loc, state->xdata);
 }
 
+static void
+fuse_link (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        struct fuse_link_in *fli = msg;
+        char         *name = (char *)(fli + 1);
+        fuse_state_t *state = NULL;
+
+        GET_STATE (this, finh, state);
+
+        fuse_resolve_inode_init (state, &state->resolve2, fli->oldnodeid);
+
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+
+        fuse_resolve_and_resume (state, fuse_link_resume);
+
+        return;
+}
 
 static int
-fuse_create_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 fd_t *fd,
-                 inode_t *inode,
-                 struct stat *buf)
+fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 fd_t *fd, inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
-        fuse_req_t req = state->req;
-        fuse_private_t *priv = this->private;
-
-        struct fuse_file_info fi = {0, };
-        struct fuse_entry_param e = {0, };
+        fuse_state_t           *state        = NULL;
+        fuse_in_header_t       *finh         = NULL;
+        fuse_private_t         *priv         = NULL;
+        struct fuse_out_header  fouh         = {0, };
+        struct fuse_entry_out   feo          = {0, };
+        struct fuse_open_out    foo          = {0, };
+        struct iovec            iov_out[3];
+        inode_t                *linked_inode = NULL;
+
+        state    = frame->root->state;
+        priv     = this->private;
+        finh     = state->finh;
+        foo.open_flags = 0;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
-        fi.flags = state->flags;
         if (op_ret >= 0) {
-                fi.fh = (unsigned long) fd;
+                foo.fh = (uintptr_t) fd;
 
-                if ((fi.flags & 3) && priv->direct_io_mode)
-                        fi.direct_io = 1;
+                if (((priv->direct_io_mode == 2)
+                     && ((state->flags & O_ACCMODE) != O_RDONLY))
+                    || (priv->direct_io_mode == 1))
+                        foo.open_flags |= FOPEN_DIRECT_IO;
 
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": %s() %s => %p (ino=%"PRId64")",
-			frame->root->unique, gf_fop_list[frame->root->op],
-			state->loc.path, fd, buf->st_ino);
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s() %s => %p (ino=%"PRIu64")",
+                        frame->root->unique, gf_fop_list[frame->root->op],
+                        state->loc.path, fd, buf->ia_ino);
 
-                e.ino = buf->st_ino;
+                buf->ia_blksize = this->ctx->page_size;
+                gf_fuse_stat2attr (buf, &feo.attr, priv->enable_ino32);
 
-#ifdef GF_DARWIN_HOST_OS
-                e.generation = 0;
-#else
-                e.generation = buf->st_ctime;
-#endif
+                linked_inode = inode_link (inode, state->loc.parent,
+                                           state->loc.name, buf);
+
+                if (linked_inode != inode) {
+                        /*
+                           VERY racy code (if used anywhere else)
+                           -- don't do this without understanding
+                        */
+                        inode_unref (fd->inode);
+                        fd->inode = inode_ref (linked_inode);
+                }
 
-                e.entry_timeout = priv->entry_timeout;
-                e.attr_timeout = priv->attribute_timeout;
-                e.attr = *buf;
-                e.attr.st_blksize = BIG_FUSE_CHANNEL_SIZE;
+                inode_lookup (linked_inode);
 
-                fi.keep_cache = 0;
+                inode_unref (linked_inode);
 
-		inode_link (inode, state->loc.parent,
-			    state->loc.name, buf);
-		
-		inode_lookup (inode);
+                feo.nodeid = inode_to_fuse_nodeid (linked_inode);
 
-		fd_ref (fd);
-                if (fuse_reply_create (req, &e, &fi) == -ENOENT) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-				"create() got EINTR");
-			inode_forget (inode, 1);
-			fd_unref (fd);
-			goto out;
-                } 
+                feo.entry_valid = calc_timeout_sec (priv->entry_timeout);
+                feo.entry_valid_nsec = calc_timeout_nsec (priv->entry_timeout);
+                feo.attr_valid = calc_timeout_sec (priv->attribute_timeout);
+                feo.attr_valid_nsec =
+                  calc_timeout_nsec (priv->attribute_timeout);
 
-		fd_bind (fd);
+                fouh.error = 0;
+                iov_out[0].iov_base = &fouh;
+                iov_out[1].iov_base = &feo;
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+                iov_out[1].iov_len = priv->proto_minor >= 9 ?
+                                     sizeof (feo) :
+                                     FUSE_COMPAT_ENTRY_OUT_SIZE;
+#else
+                iov_out[1].iov_len = sizeof (feo);
+#endif
+                iov_out[2].iov_base = &foo;
+                iov_out[2].iov_len = sizeof (foo);
+
+                if (send_fuse_iov (this, finh, iov_out, 3) == ENOENT) {
+                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+                                "create(%s) got EINTR", state->loc.path);
+                        inode_forget (inode, 1);
+                        gf_fd_put (priv->fdtable, state->fd_no);
+                        goto out;
+                }
+
+                fd_bind (fd);
         } else {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": %s => -1 (%s)", req_callid (req),
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": %s => -1 (%s)", finh->unique,
                         state->loc.path, strerror (op_errno));
-                fuse_reply_err (req, op_errno);
+                send_fuse_err (this, finh, op_errno);
+                gf_fd_put (priv->fdtable, state->fd_no);
         }
 out:
-        free_state (state);
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
-
-static void
-fuse_create (fuse_req_t req,
-             fuse_ino_t par,
-             const char *name,
-             mode_t mode,
-             struct fuse_file_info *fi)
+void
+fuse_create_resume (fuse_state_t *state)
 {
-        fuse_state_t *state;
-        fd_t *fd;
-	int32_t ret = -1;
-
-        state = state_from_req (req);
-        state->flags = fi->flags;
-
-        ret = fuse_loc_fill (&state->loc, state, 0, par, name);
-	if (ret < 0) {
-		gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64" CREATE %s (fuse_loc_fill() failed)", 
-                        req_callid (req), state->loc.path);
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
+
+        if (!state->loc.parent) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64" CREATE %s/%s resolution failed",
+                        state->finh->unique, uuid_utoa (state->resolve.gfid),
+                        state->resolve.bname);
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
-	}
+        }
 
-        state->loc.inode = inode_new (state->itable);
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
 
-        fd = fd_create (state->loc.inode, get_pid_from_req (req));
-        state->fd = fd;
-	fd->flags = state->flags;
+        if (state->loc.inode) {
+                gf_log (state->this->name, GF_LOG_DEBUG,
+                        "inode already present");
+                inode_unref (state->loc.inode);
+        }
+
+        state->loc.inode = inode_new (state->loc.parent->table);
+
+        fd = fd_create (state->loc.inode, state->finh->pid);
+        if (fd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64" CREATE cannot create a new fd",
+                        state->finh->unique);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64" CREATE creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": CREATE %s", req_callid (req),
+        priv = state->this->private;
+
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+
+        state->fd = fd_ref (fd);
+        fd->flags = state->flags;
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": CREATE %s", state->finh->unique,
                 state->loc.path);
 
         FUSE_FOP (state, fuse_create_cbk, GF_FOP_CREATE,
-                  create, &state->loc, state->flags, mode, fd);
+                  create, &state->loc, state->flags, state->mode,
+                  state->umask, fd, state->xdata);
 
-        return;
 }
 
-
 static void
-fuse_open (fuse_req_t req,
-           fuse_ino_t ino,
-           struct fuse_file_info *fi)
+fuse_create (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-        fd_t *fd;
-	int32_t ret = -1;
+#if FUSE_KERNEL_MINOR_VERSION >= 12
+        struct fuse_create_in *fci = msg;
+#else
+        struct fuse_open_in *fci = msg;
+#endif
+        char         *name = (char *)(fci + 1);
+
+        fuse_private_t        *priv = NULL;
+        fuse_state_t *state = NULL;
+        int32_t       ret = -1;
+
+        priv = this->private;
+#if FUSE_KERNEL_MINOR_VERSION >= 12
+        if (priv->proto_minor < 12)
+                name = (char *)((struct fuse_open_in *)msg + 1);
+#endif
+
+        GET_STATE (this, finh, state);
 
-        state = state_from_req (req);
-        state->flags = fi->flags;
+        uuid_generate (state->gfid);
 
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
+        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+
+        state->mode = fci->mode;
+        state->flags = fci->flags;
+
+        priv = this->private;
+#if FUSE_KERNEL_MINOR_VERSION >=12
+        FUSE_ENTRY_CREATE(this, priv, finh, state, fci, "CREATE");
+#endif
+        fuse_resolve_and_resume (state, fuse_create_resume);
+
+        return;
+}
+
+void
+fuse_open_resume (fuse_state_t *state)
+{
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
+
+        if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": OPEN %s (fuse_loc_fill() failed)",
-                        req_callid (req), state->loc.path);
-  
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+                        "%"PRIu64": OPEN %s resolution failed",
+                        state->finh->unique, uuid_utoa (state->resolve.gfid));
+
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
+                return;
+        }
+
+        fd = fd_create (state->loc.inode, state->finh->pid);
+        if (!fd) {
+                gf_log ("fuse", GF_LOG_ERROR,
+                        "fd is NULL");
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
 
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPEN creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
 
-        fd = fd_create (state->loc.inode, get_pid_from_req (req));
-        state->fd = fd;
-	fd->flags = fi->flags;
+        priv = state->this->private;
+
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+        state->fd = fd_ref (fd);
+        fd->flags = state->flags;
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": OPEN %s", req_callid (req),
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": OPEN %s", state->finh->unique,
                 state->loc.path);
 
         FUSE_FOP (state, fuse_fd_cbk, GF_FOP_OPEN,
-                  open, &state->loc, fi->flags, fd);
+                  open, &state->loc, state->flags, fd, state->xdata);
+}
+
+static void
+fuse_open (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        struct fuse_open_in *foi = msg;
+        fuse_state_t *state = NULL;
+
+        GET_STATE (this, finh, state);
+
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+
+        state->flags = foi->flags;
+
+        fuse_resolve_and_resume (state, fuse_open_resume);
 
         return;
 }
 
-
 static int
-fuse_readv_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                struct iovec *vector,
-                int32_t count,
-                struct stat *stbuf)
+fuse_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno,
+                struct iovec *vector, int32_t count,
+                struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
-        fuse_req_t req = state->req;
+        fuse_state_t *state = NULL;
+        fuse_in_header_t *finh = NULL;
+        struct fuse_out_header fouh = {0, };
+        struct iovec *iov_out = NULL;
 
-        if (op_ret >= 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": READ => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRId64,
-			frame->root->unique,
-                        op_ret, state->size, state->off, stbuf->st_size);
+        state = frame->root->state;
+        finh = state->finh;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
-                fuse_reply_vec (req, vector, count);
+        if (op_ret >= 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": READ => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRIu64,
+                        frame->root->unique,
+                        op_ret, state->size, state->off, stbuf->ia_size);
+
+                iov_out = GF_CALLOC (count + 1, sizeof (*iov_out),
+                                     gf_fuse_mt_iovec);
+                if (iov_out) {
+                        fouh.error = 0;
+                        iov_out[0].iov_base = &fouh;
+                        memcpy (iov_out + 1, vector, count * sizeof (*iov_out));
+                        send_fuse_iov (this, finh, iov_out, count + 1);
+                        GF_FREE (iov_out);
+                } else
+                        send_fuse_err (this, finh, ENOMEM);
         } else {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": READ => %d (%s)", frame->root->unique, 
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": READ => %d (%s)", frame->root->unique,
                         op_ret, strerror (op_errno));
 
-                fuse_reply_err (req, op_errno);
+                send_fuse_err (this, finh, op_errno);
         }
-        
-        free_state (state);
+
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
+void
+fuse_readv_resume (fuse_state_t *state)
+{
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": READ (%p, size=%zu, offset=%"PRIu64")",
+                state->finh->unique, state->fd, state->size, state->off);
+
+        FUSE_FOP (state, fuse_readv_cbk, GF_FOP_READ, readv, state->fd,
+                  state->size, state->off, state->io_flags, state->xdata);
+}
+
 static void
-fuse_readv (fuse_req_t req,
-            fuse_ino_t ino,
-            size_t size,
-            off_t off,
-            struct fuse_file_info *fi)
+fuse_readv (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-        fd_t *fd = NULL;
-        state = state_from_req (req);
-        state->size = size;
-        state->off = off;
-        
-        fd = FI_TO_FD (fi);
+        struct fuse_read_in *fri = msg;
+
+        fuse_private_t  *priv = NULL;
+        fuse_state_t *state = NULL;
+        fd_t         *fd = NULL;
+
+        GET_STATE (this, finh, state);
+
+        fd = FH_TO_FD (fri->fh);
         state->fd = fd;
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": READ (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-                req_callid (req), fd, size, off);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
-        FUSE_FOP (state, fuse_readv_cbk, GF_FOP_READ,
-                  readv, fd, size, off);
+        /* See comment by similar code in fuse_settatr */
+        priv = this->private;
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+        if (priv->proto_minor >= 9 && fri->read_flags & FUSE_READ_LOCKOWNER)
+                state->lk_owner = fri->lock_owner;
+#endif
 
-}
+        state->size = fri->size;
+        state->off = fri->offset;
+        /* lets ignore 'fri->read_flags', but just consider 'fri->flags' */
+        state->io_flags = fri->flags;
 
+        fuse_resolve_and_resume (state, fuse_readv_resume);
+}
 
 static int
-fuse_writev_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct stat *stbuf)
+fuse_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 struct iatt *stbuf, struct iatt *postbuf, dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
-        fuse_req_t req = state->req;
+        fuse_state_t *state = NULL;
+        fuse_in_header_t *finh = NULL;
+        struct fuse_write_out fwo = {0, };
+
+        state = frame->root->state;
+        finh = state->finh;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
         if (op_ret >= 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": WRITE => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRId64,
-			frame->root->unique,
-                        op_ret, state->size, state->off, stbuf->st_size);
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": WRITE => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRIu64,
+                        frame->root->unique,
+                        op_ret, state->size, state->off, stbuf->ia_size);
 
-                fuse_reply_write (req, op_ret);
+                fwo.size = op_ret;
+                send_fuse_obj (this, finh, &fwo);
         } else {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": WRITE => -1 (%s)", frame->root->unique, 
-                        strerror(op_errno));
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": WRITE => -1 (%s)", frame->root->unique,
+                        strerror (op_errno));
 
-                fuse_reply_err (req, op_errno);
+                send_fuse_err (this, finh, op_errno);
         }
-        
-        free_state (state);
+
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
+void
+fuse_write_resume (fuse_state_t *state)
+{
+        struct iobref *iobref = NULL;
+        struct iobuf  *iobuf = NULL;
+
+
+        iobref = iobref_new ();
+        if (!iobref) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "%"PRIu64": WRITE iobref allocation failed",
+                        state->finh->unique);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+
+                free_fuse_state (state);
+                return;
+        }
+
+        iobuf = ((fuse_private_t *) (state->this->private))->iobuf;
+        iobref_add (iobref, iobuf);
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": WRITE (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
+                state->finh->unique, state->fd, state->size, state->off);
+
+        FUSE_FOP (state, fuse_writev_cbk, GF_FOP_WRITE, writev, state->fd,
+                  &state->vector, 1, state->off, state->io_flags, iobref,
+                  state->xdata);
+
+        iobref_unref (iobref);
+}
 
 static void
-fuse_write (fuse_req_t req,
-            fuse_ino_t ino,
-            const char *buf,
-            size_t size,
-            off_t off,
-            struct fuse_file_info *fi)
+fuse_write (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-        struct iovec vector;
-	fd_t *fd = NULL;
+        /* WRITE is special, metadata is attached to in_header,
+         * and msg is the payload as-is.
+         */
+        struct fuse_write_in *fwi = (struct fuse_write_in *)
+                                      (finh + 1);
+
+        fuse_private_t  *priv = NULL;
+        fuse_state_t    *state = NULL;
+        fd_t            *fd = NULL;
+
+        priv = this->private;
+
+        GET_STATE (this, finh, state);
+        fd          = FH_TO_FD (fwi->fh);
+        state->fd   = fd;
+        state->size = fwi->size;
+        state->off  = fwi->offset;
+
+        /* lets ignore 'fwi->write_flags', but just consider 'fwi->flags' */
+        state->io_flags = fwi->flags;
+        /* TODO: may need to handle below flag
+           (fwi->write_flags & FUSE_WRITE_CACHE);
+        */
 
-        state = state_from_req (req);
-        state->size = size;
-        state->off = off;
-	fd = FI_TO_FD (fi);
-	state->fd = fd;
-        vector.iov_base = (void *)buf;
-        vector.iov_len = size;
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": WRITE (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-                req_callid (req), fd, size, off);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
+
+        /* See comment by similar code in fuse_settatr */
+        priv = this->private;
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+        if (priv->proto_minor >= 9 && fwi->write_flags & FUSE_WRITE_LOCKOWNER)
+                state->lk_owner = fwi->lock_owner;
+#endif
+
+        state->vector.iov_base = msg;
+        state->vector.iov_len  = fwi->size;
+
+        fuse_resolve_and_resume (state, fuse_write_resume);
 
-        FUSE_FOP (state, fuse_writev_cbk, GF_FOP_WRITE,
-                  writev, fd, &vector, 1, off);
         return;
 }
 
+void
+fuse_flush_resume (fuse_state_t *state)
+{
+        FUSE_FOP (state, fuse_err_cbk, GF_FOP_FLUSH,
+                  flush, state->fd, state->xdata);
+}
 
 static void
-fuse_flush (fuse_req_t req,
-            fuse_ino_t ino,
-            struct fuse_file_info *fi)
+fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	fd_t *fd = NULL;
+        struct fuse_flush_in *ffi = msg;
 
-        state = state_from_req (req);
-	fd = FI_TO_FD (fi);
-	state->fd = fd;
+        fuse_state_t *state = NULL;
+        fd_t         *fd = NULL;
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": FLUSH %p", req_callid (req), fd);
+        GET_STATE (this, finh, state);
+        fd = FH_TO_FD (ffi->fh);
+        state->fd = fd;
 
-        FUSE_FOP (state, fuse_err_cbk, GF_FOP_FLUSH,
-                  flush, fd);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
+
+        state->lk_owner = ffi->lock_owner;
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": FLUSH %p", finh->unique, fd);
+
+        fuse_resolve_and_resume (state, fuse_flush_resume);
 
         return;
 }
 
-
-static void 
-fuse_release (fuse_req_t req,
-	      fuse_ino_t ino,
-	      struct fuse_file_info *fi)
+static void
+fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
+        struct fuse_release_in *fri       = msg;
+        fd_t                   *activefd = NULL;
+        fd_t                   *fd        = NULL;
+        uint64_t                val       = 0;
+        int                     ret       = 0;
+        fuse_state_t           *state     = NULL;
+        fuse_fd_ctx_t          *fdctx     = NULL;
+        fuse_private_t         *priv      = NULL;
+
+        GET_STATE (this, finh, state);
+        fd = FH_TO_FD (fri->fh);
+        state->fd = fd;
 
-        state = state_from_req (req);
-        state->fd = FI_TO_FD (fi);
+        priv = this->private;
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": RELEASE %p", req_callid (req), state->fd);
+        fuse_log_eh (this, "RELEASE(): %"PRIu64":, fd: %p, gfid: %s",
+                     finh->unique, fd, uuid_utoa (fd->inode->gfid));
 
-        fd_unref (state->fd);
-        
-        fuse_reply_err (req, 0);
-        
-        free_state (state);
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": RELEASE %p", finh->unique, state->fd);
+
+        ret = fd_ctx_del (fd, this, &val);
+        if (!ret) {
+                fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
+                if (fdctx) {
+                        activefd = fdctx->activefd;
+                        if (activefd) {
+                                fd_unref (activefd);
+                        }
+
+                        GF_FREE (fdctx);
+                }
+        }
+        fd_unref (fd);
+
+        state->fd = NULL;
+
+        gf_fdptr_put (priv->fdtable, fd);
+
+        send_fuse_err (this, finh, 0);
+
+        free_fuse_state (state);
         return;
 }
 
+void
+fuse_fsync_resume (fuse_state_t *state)
+{
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": FSYNC %p", state->finh->unique,
+                state->fd);
 
-static void 
-fuse_fsync (fuse_req_t req,
-            fuse_ino_t ino,
-            int datasync,
-            struct fuse_file_info *fi)
+        /* fsync_flags: 1 means "datasync" (no defines for this) */
+        FUSE_FOP (state, fuse_fsync_cbk, GF_FOP_FSYNC,
+                  fsync, state->fd, (state->flags & 1), state->xdata);
+}
+
+static void
+fuse_fsync (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-        fd_t *fd = NULL;
+        struct fuse_fsync_in *fsi = msg;
 
-        state = state_from_req (req);
-        fd = FI_TO_FD (fi);
-        state->fd = fd;
+        fuse_state_t *state = NULL;
+        fd_t         *fd = NULL;
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": FSYNC %p", req_callid (req), fd);
+        GET_STATE (this, finh, state);
+        fd = FH_TO_FD (fsi->fh);
+        state->fd = fd;
 
-        FUSE_FOP (state, fuse_err_cbk, GF_FOP_FSYNC,
-                  fsync, fd, datasync);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
+        state->flags = fsi->fsync_flags;
+        fuse_resolve_and_resume (state, fuse_fsync_resume);
         return;
 }
 
-
-static void
-fuse_opendir (fuse_req_t req,
-              fuse_ino_t ino,
-              struct fuse_file_info *fi)
+void
+fuse_opendir_resume (fuse_state_t *state)
 {
-        fuse_state_t *state;
-        fd_t *fd;
-	int32_t ret = -1;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
 
-        state = state_from_req (req);
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": OPENDIR %s (fuse_loc_fill() failed)",
-                        req_callid (req), state->loc.path);
-  
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+        priv = state->this->private;
+
+        if (!state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPENDIR (%s) resolution failed",
+                        state->finh->unique, uuid_utoa (state->resolve.gfid));
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
 
-        fd = fd_create (state->loc.inode, get_pid_from_req (req));
-        state->fd = fd;
+        fd = fd_create (state->loc.inode, state->finh->pid);
+        if (fd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPENDIR fd creation failed",
+                        state->finh->unique);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+        }
+
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPENDIR creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": OPENDIR %s", req_callid (req),
+        state->fd = fd_ref (fd);
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": OPENDIR %s", state->finh->unique,
                 state->loc.path);
 
         FUSE_FOP (state, fuse_fd_cbk, GF_FOP_OPENDIR,
-                  opendir, &state->loc, fd);
+                  opendir, &state->loc, fd, state->xdata);
+}
+
+static void
+fuse_opendir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        /*
+        struct fuse_open_in *foi = msg;
+         */
+
+        fuse_state_t *state = NULL;
+
+        GET_STATE (this, finh, state);
+
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+
+        fuse_resolve_and_resume (state, fuse_opendir_resume);
+}
+
+unsigned char
+d_type_from_stat (struct iatt *buf)
+{
+        unsigned char d_type;
+
+        if (IA_ISLNK (buf->ia_type)) {
+                d_type = DT_LNK;
+
+        } else if (IA_ISDIR (buf->ia_type)) {
+                d_type = DT_DIR;
+
+        } else if (IA_ISFIFO (buf->ia_type)) {
+                d_type = DT_FIFO;
+
+        } else if (IA_ISSOCK (buf->ia_type)) {
+                d_type = DT_SOCK;
+
+        } else if (IA_ISCHR (buf->ia_type)) {
+                d_type = DT_CHR;
+
+        } else if (IA_ISBLK (buf->ia_type)) {
+                d_type = DT_BLK;
+
+        } else if (IA_ISREG (buf->ia_type)) {
+                d_type = DT_REG;
+
+        } else {
+                d_type = DT_UNKNOWN;
+        }
+
+        return d_type;
 }
 
 static int
-fuse_readdir_cbk (call_frame_t *frame,
-                  void *cookie,
-                  xlator_t *this,
-                  int32_t op_ret,
-                  int32_t op_errno,
-                  gf_dirent_t *entries)
+fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
-        fuse_req_t    req = state->req;
+        fuse_state_t *state = NULL;
+        fuse_in_header_t *finh = NULL;
+        int           size = 0;
+        char         *buf = NULL;
+        gf_dirent_t  *entry = NULL;
+        struct fuse_dirent *fde = NULL;
+        fuse_private_t *priv = NULL;
+
+        state = frame->root->state;
+        finh  = state->finh;
+        priv = state->this->private;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+        if (op_ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": READDIR => -1 (%s)", frame->root->unique,
+                        strerror (op_errno));
+
+                send_fuse_err (this, finh, op_errno);
+                goto out;
+        }
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": READDIR => %d/%"GF_PRI_SIZET",%"PRId64,
+                frame->root->unique, op_ret, state->size, state->off);
+
+        list_for_each_entry (entry, &entries->list, list) {
+                size += FUSE_DIRENT_ALIGN (FUSE_NAME_OFFSET +
+                                           strlen (entry->d_name));
+        }
+
+	if (size <= 0) {
+		send_fuse_data (this, finh, 0, 0);
+		goto out;
+	}
+
+        buf = GF_CALLOC (1, size, gf_fuse_mt_char);
+        if (!buf) {
+                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+                        "%"PRIu64": READDIR => -1 (%s)", frame->root->unique,
+                        strerror (ENOMEM));
+                send_fuse_err (this, finh, ENOMEM);
+                goto out;
+        }
+
+        size = 0;
+        list_for_each_entry (entry, &entries->list, list) {
+                fde = (struct fuse_dirent *)(buf + size);
+                gf_fuse_fill_dirent (entry, fde, priv->enable_ino32);
+                size += FUSE_DIRENT_SIZE (fde);
+        }
+
+        send_fuse_data (this, finh, buf, size);
+
+        /* TODO: */
+        /* gf_link_inodes_from_dirent (this, state->fd->inode, entries); */
+
+out:
+        free_fuse_state (state);
+        STACK_DESTROY (frame->root);
+        GF_FREE (buf);
+        return 0;
+
+}
+
+void
+fuse_readdir_resume (fuse_state_t *state)
+{
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": READDIR (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
+                state->finh->unique, state->fd, state->size, state->off);
+
+        FUSE_FOP (state, fuse_readdir_cbk, GF_FOP_READDIR,
+                  readdir, state->fd, state->size, state->off, state->xdata);
+}
+
+static void
+fuse_readdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        struct fuse_read_in *fri = msg;
+
+        fuse_state_t *state = NULL;
+        fd_t         *fd = NULL;
+
+        GET_STATE (this, finh, state);
+        state->size = fri->size;
+        state->off = fri->offset;
+        fd = FH_TO_FD (fri->fh);
+        state->fd = fd;
+
+        fuse_resolve_fd_init (state, &state->resolve, fd);
+
+        fuse_resolve_and_resume (state, fuse_readdir_resume);
+}
+
+
+static int
+fuse_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+		   dict_t *xdata)
+{
+	fuse_state_t *state = NULL;
+	fuse_in_header_t *finh = NULL;
 	int           size = 0;
-	int           entry_size = 0;
 	char         *buf = NULL;
 	gf_dirent_t  *entry = NULL;
-	struct stat   stbuf = {0, };
+	struct fuse_direntplus *fde = NULL;
+	struct fuse_entry_out *feo = NULL;
+	fuse_private_t         *priv   = NULL;
 
-        if (op_ret < 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": READDIR => -1 (%s)", frame->root->unique, 
-                        strerror (op_errno));
+	state = frame->root->state;
+	finh  = state->finh;
+	priv = this->private;
+
+	if (op_ret < 0) {
+		gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+			"%"PRIu64": READDIRP => -1 (%s)", frame->root->unique,
+			strerror (op_errno));
 
-                fuse_reply_err (req, op_errno);
+		send_fuse_err (this, finh, op_errno);
 		goto out;
 	}
 
-	gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-		"%"PRId64": READDIR => %d/%"GF_PRI_SIZET",%"PRId64,
+	gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+		"%"PRIu64": READDIRP => %d/%"GF_PRI_SIZET",%"PRId64,
 		frame->root->unique, op_ret, state->size, state->off);
 
 	list_for_each_entry (entry, &entries->list, list) {
-		size += fuse_dirent_size (strlen (entry->d_name));
+		size += FUSE_DIRENT_ALIGN (FUSE_NAME_OFFSET_DIRENTPLUS +
+					   strlen (entry->d_name));
 	}
 
-	buf = CALLOC (1, size);
+	if (size <= 0) {
+		send_fuse_data (this, finh, 0, 0);
+		goto out;
+	}
+
+	buf = GF_CALLOC (1, size, gf_fuse_mt_char);
 	if (!buf) {
-		gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-			"%"PRId64": READDIR => -1 (%s)", frame->root->unique,
+		gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+			"%"PRIu64": READDIRP => -1 (%s)", frame->root->unique,
 			strerror (ENOMEM));
-		fuse_reply_err (req, -ENOMEM);
+		send_fuse_err (this, finh, ENOMEM);
 		goto out;
 	}
 
 	size = 0;
 	list_for_each_entry (entry, &entries->list, list) {
-		stbuf.st_ino = entry->d_ino;
-		entry_size = fuse_dirent_size (strlen (entry->d_name));
-		fuse_add_direntry (req, buf + size, entry_size,
-				   entry->d_name, &stbuf,
-				   entry->d_off);
-		size += entry_size;
-	}
+		inode_t *linked_inode;
+
+		fde = (struct fuse_direntplus *)(buf + size);
+		feo = &fde->entry_out;
+		fde->dirent.ino = entry->d_ino;
+		fde->dirent.off = entry->d_off;
+		fde->dirent.type = entry->d_type;
+		fde->dirent.namelen = strlen (entry->d_name);
+		strncpy (fde->dirent.name, entry->d_name, fde->dirent.namelen);
+		size += FUSE_DIRENTPLUS_SIZE (fde);
+
+		if (!entry->inode)
+			continue;
+
+		entry->d_stat.ia_blksize = this->ctx->page_size;
+		gf_fuse_stat2attr (&entry->d_stat, &feo->attr, priv->enable_ino32);
+
+		linked_inode = inode_link (entry->inode, state->fd->inode,
+					   entry->d_name, &entry->d_stat);
+		if (!linked_inode)
+			continue;
+
+		inode_lookup (linked_inode);
+
+		feo->nodeid = inode_to_fuse_nodeid (linked_inode);
 
-	fuse_reply_buf (req, (void *)buf, size);
+		fuse_inode_set_need_lookup (linked_inode, this);
 
+		inode_unref (linked_inode);
+
+		feo->entry_valid =
+			calc_timeout_sec (priv->entry_timeout);
+		feo->entry_valid_nsec =
+			calc_timeout_nsec (priv->entry_timeout);
+		feo->attr_valid =
+			calc_timeout_sec (priv->attribute_timeout);
+		feo->attr_valid_nsec =
+			calc_timeout_nsec (priv->attribute_timeout);
+	}
+
+	send_fuse_data (this, finh, buf, size);
 out:
-        free_state (state);
-        STACK_DESTROY (frame->root);
-	if (buf)
-		FREE (buf);
-        return 0;
+	free_fuse_state (state);
+	STACK_DESTROY (frame->root);
+	GF_FREE (buf);
+	return 0;
 
 }
 
+
+void
+fuse_readdirp_resume (fuse_state_t *state)
+{
+	gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+		"%"PRIu64": READDIRP (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
+		state->finh->unique, state->fd, state->size, state->off);
+
+	FUSE_FOP (state, fuse_readdirp_cbk, GF_FOP_READDIRP,
+		  readdirp, state->fd, state->size, state->off, state->xdata);
+}
+
+
 static void
-fuse_readdir (fuse_req_t req,
-              fuse_ino_t ino,
-              size_t size,
-              off_t off,
-              struct fuse_file_info *fi)
+fuse_readdirp (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	fd_t *fd = NULL;
+	struct fuse_read_in *fri = msg;
 
-        state = state_from_req (req);
-        state->size = size;
-        state->off = off;
-	fd = FI_TO_FD (fi);
+	fuse_state_t *state = NULL;
+	fd_t         *fd = NULL;
+
+	GET_STATE (this, finh, state);
+	state->size = fri->size;
+	state->off = fri->offset;
+	fd = FH_TO_FD (fri->fh);
 	state->fd = fd;
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": READDIR (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-                req_callid (req), fd, size, off);
+	fuse_resolve_fd_init (state, &state->resolve, fd);
 
-        FUSE_FOP (state, fuse_readdir_cbk, GF_FOP_READDIR,
-                  readdir, fd, size, off);
+	fuse_resolve_and_resume (state, fuse_readdirp_resume);
 }
 
+static int
+fuse_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		   struct iatt *postbuf, dict_t *xdata)
+{
+	return fuse_err_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+}
 
-static void 
-fuse_releasedir (fuse_req_t req,
-		 fuse_ino_t ino,
-		 struct fuse_file_info *fi)
+static void
+fuse_fallocate_resume(fuse_state_t *state)
 {
-        fuse_state_t *state;
+	gf_log("glusterfs-fuse", GF_LOG_TRACE,
+	       "%"PRIu64": FALLOCATE (%p, flags=%d, size=%zu, offset=%"PRId64")",
+	       state->finh->unique, state->fd, state->flags, state->size,
+	       state->off);
+
+	if (state->flags & FALLOC_FL_PUNCH_HOLE)
+		FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_DISCARD, discard,
+			 state->fd, state->off, state->size, state->xdata);
+	else
+		FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_FALLOCATE, fallocate,
+			 state->fd, (state->flags & FALLOC_FL_KEEP_SIZE),
+			 state->off, state->size, state->xdata);
+}
+
+static void
+fuse_fallocate(xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+	struct fuse_fallocate_in *ffi = msg;
+	fuse_state_t *state = NULL;
+
+	GET_STATE(this, finh, state);
+	state->off = ffi->offset;
+	state->size = ffi->length;
+	state->flags = ffi->mode;
+	state->fd = FH_TO_FD(ffi->fh);
+
+	fuse_resolve_fd_init(state, &state->resolve, state->fd);
+	fuse_resolve_and_resume(state, fuse_fallocate_resume);
+}
 
-        state = state_from_req (req);
-        state->fd = FI_TO_FD (fi);
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": RELEASEDIR %p", req_callid (req), state->fd);
-	
-	fd_unref (state->fd);
+static void
+fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        struct fuse_release_in *fri       = msg;
+        fd_t                   *activefd = NULL;
+        uint64_t                val       = 0;
+        int                     ret       = 0;
+        fuse_state_t           *state     = NULL;
+        fuse_fd_ctx_t          *fdctx     = NULL;
+        fuse_private_t         *priv      = NULL;
+
+        GET_STATE (this, finh, state);
+        state->fd = FH_TO_FD (fri->fh);
+
+        priv = this->private;
+
+        fuse_log_eh (this, "RELEASEDIR (): %"PRIu64": fd: %p, gfid: %s",
+                     finh->unique, state->fd,
+                     uuid_utoa (state->fd->inode->gfid));
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": RELEASEDIR %p", finh->unique, state->fd);
+
+        ret = fd_ctx_del (state->fd, this, &val);
+
+        if (!ret) {
+                fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
+                if (fdctx) {
+                        activefd = fdctx->activefd;
+                        if (activefd) {
+                                fd_unref (activefd);
+                        }
 
-	fuse_reply_err (req, 0);
-	
-	free_state (state);
+                        GF_FREE (fdctx);
+                }
+        }
+
+        fd_unref (state->fd);
+
+        gf_fdptr_put (priv->fdtable, state->fd);
+
+        state->fd = NULL;
+
+        send_fuse_err (this, finh, 0);
+
+        free_fuse_state (state);
 
         return;
 }
 
+void
+fuse_fsyncdir_resume (fuse_state_t *state)
+{
+        FUSE_FOP (state, fuse_err_cbk, GF_FOP_FSYNCDIR,
+                  fsyncdir, state->fd, (state->flags & 1), state->xdata);
+
+}
 
-static void 
-fuse_fsyncdir (fuse_req_t req,
-               fuse_ino_t ino,
-               int datasync,
-               struct fuse_file_info *fi)
+static void
+fuse_fsyncdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	fd_t *fd = NULL;
-	
-	fd = FI_TO_FD (fi);
+        struct fuse_fsync_in *fsi = msg;
 
-        state = state_from_req (req);
-	state->fd = fd;
+        fuse_state_t *state = NULL;
+        fd_t         *fd = NULL;
 
-        FUSE_FOP (state, fuse_err_cbk, GF_FOP_FSYNCDIR,
-                  fsyncdir, fd, datasync);
+        fd = FH_TO_FD (fsi->fh);
+
+        GET_STATE (this, finh, state);
+        state->fd = fd;
+
+        fuse_resolve_fd_init (state, &state->resolve, fd);
+
+        state->flags = fsi->fsync_flags;
+        fuse_resolve_and_resume (state, fuse_fsyncdir_resume);
 
         return;
 }
 
-
 static int
-fuse_statfs_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct statvfs *buf)
+fuse_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                 dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
-        fuse_req_t req = state->req;
+        fuse_state_t *state = NULL;
+        fuse_in_header_t *finh = NULL;
+        fuse_private_t   *priv = NULL;
+        struct fuse_statfs_out fso = {{0, }, };
 
-        /*
-          Filesystems (like ZFS on solaris) reports
-          different ->f_frsize and ->f_bsize. Old coreutils
-          df tools use statfs() and do not see ->f_frsize.
-          the ->f_blocks, ->f_bavail and ->f_bfree are
-          w.r.t ->f_frsize and not ->f_bsize which makes the
-          df tools report wrong values.
-
-          Scale the block counts to match ->f_bsize.
-        */
-        /* TODO: with old coreutils, f_bsize is taken from stat()'s st_blksize
-         * so the df with old coreutils this wont work :(
-         */
+        state = frame->root->state;
+        priv  = this->private;
+        finh  = state->finh;
+
+        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s()",
+                     op_ret, op_errno, frame->root->unique,
+                     gf_fop_list[frame->root->op]);
 
         if (op_ret == 0) {
 #ifndef GF_DARWIN_HOST_OS
                 /* MacFUSE doesn't respect anyof these tweaks */
                 buf->f_blocks *= buf->f_frsize;
-                buf->f_blocks /= BIG_FUSE_CHANNEL_SIZE;
+                buf->f_blocks /= this->ctx->page_size;
 
                 buf->f_bavail *= buf->f_frsize;
-                buf->f_bavail /= BIG_FUSE_CHANNEL_SIZE;
+                buf->f_bavail /= this->ctx->page_size;
 
                 buf->f_bfree *= buf->f_frsize;
-                buf->f_bfree /= BIG_FUSE_CHANNEL_SIZE;
+                buf->f_bfree /= this->ctx->page_size;
 
-                buf->f_frsize = buf->f_bsize = BIG_FUSE_CHANNEL_SIZE;
+                buf->f_frsize = buf->f_bsize =this->ctx->page_size;
 #endif /* GF_DARWIN_HOST_OS */
-                fuse_reply_statfs (req, buf);
-
+                fso.st.bsize   = buf->f_bsize;
+                fso.st.frsize  = buf->f_frsize;
+                fso.st.blocks  = buf->f_blocks;
+                fso.st.bfree   = buf->f_bfree;
+                fso.st.bavail  = buf->f_bavail;
+                fso.st.files   = buf->f_files;
+                fso.st.ffree   = buf->f_ffree;
+                fso.st.namelen = buf->f_namemax;
+
+                priv->proto_minor >= 4 ?
+                send_fuse_obj (this, finh, &fso) :
+                send_fuse_data (this, finh, &fso, FUSE_COMPAT_STATFS_SIZE);
         } else {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": ERR => -1 (%s)", frame->root->unique, 
-                        strerror(op_errno));
-                fuse_reply_err (req, op_errno);
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": ERR => -1 (%s)", frame->root->unique,
+                        strerror (op_errno));
+                send_fuse_err (this, finh, op_errno);
         }
 
-        free_state (state);
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
+void
+fuse_statfs_resume (fuse_state_t *state)
+{
+        if (!state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": STATFS (%s) resolution fail",
+                        state->finh->unique, uuid_utoa (state->resolve.gfid));
+
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
+                return;
+        }
+
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": STATFS", state->finh->unique);
+
+        FUSE_FOP (state, fuse_statfs_cbk, GF_FOP_STATFS,
+                  statfs, &state->loc, state->xdata);
+}
+
 
 static void
-fuse_statfs (fuse_req_t req,
-             fuse_ino_t ino)
+fuse_statfs (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
+        fuse_state_t *state = NULL;
 
-        state = state_from_req (req);
-        ret = fuse_loc_fill (&state->loc, state, 1, 0, NULL);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": STATFS (fuse_loc_fill() fail)",
-			req_callid (req));
-    
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+        GET_STATE (this, finh, state);
+
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+
+        fuse_resolve_and_resume (state, fuse_statfs_resume);
+}
+
+
+void
+fuse_setxattr_resume (fuse_state_t *state)
+{
+        if (!state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": SETXATTR %s/%"PRIu64" (%s) "
+                        "resolution failed",
+                        state->finh->unique, uuid_utoa (state->resolve.gfid),
+                        state->finh->nodeid, state->name);
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": STATFS", req_callid (req));
+#ifdef GF_TEST_FFOP
+        state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+#endif /* GF_TEST_FFOP */
 
-        FUSE_FOP (state, fuse_statfs_cbk, GF_FOP_STATFS,
-                  statfs, &state->loc);
+        if (state->fd) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": SETXATTR %p/%"PRIu64" (%s)", state->finh->unique,
+                        state->fd, state->finh->nodeid, state->name);
+
+                FUSE_FOP (state, fuse_setxattr_cbk, GF_FOP_FSETXATTR,
+                          fsetxattr, state->fd, state->xattr, state->flags,
+                          state->xdata);
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": SETXATTR %s/%"PRIu64" (%s)", state->finh->unique,
+                        state->loc.path, state->finh->nodeid, state->name);
+
+                FUSE_FOP (state, fuse_setxattr_cbk, GF_FOP_SETXATTR,
+                          setxattr, &state->loc, state->xattr, state->flags,
+                          state->xdata);
+        }
 }
 
 
 static void
-fuse_setxattr (fuse_req_t req,
-               fuse_ino_t ino,
-               const char *name,
-               const char *value,
-               size_t size,
-               int flags)
+fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	char *dict_value = NULL;
-	int32_t ret = -1;
+        struct fuse_setxattr_in *fsi = msg;
+        char         *name = (char *)(fsi + 1);
+        char         *value = name + strlen (name) + 1;
+        struct fuse_private *priv = NULL;
 
-#ifdef DISABLE_POSIX_ACL
-	if (!strncmp (name, "system.", 7)) {
-		fuse_reply_err (req, EOPNOTSUPP);
-		return;
-	}
+        fuse_state_t *state = NULL;
+        char         *dict_value = NULL;
+        int32_t       ret = -1;
+        char *newkey = NULL;
+
+        priv = this->private;
+
+#ifdef GF_DARWIN_HOST_OS
+        if (fsi->position) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": SETXATTR %s/%"PRIu64" (%s):"
+                        "refusing positioned setxattr",
+                        finh->unique, state->loc.path, finh->nodeid, name);
+                send_fuse_err (this, finh, EINVAL);
+                FREE (finh);
+                return;
+        }
 #endif
 
-        state = state_from_req (req);
-        state->size = size;
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
+        if (fuse_ignore_xattr_set (priv, name)) {
+                (void) send_fuse_err (this, finh, 0);
+                return;
+        }
+
+        if (!priv->acl) {
+                if ((strcmp (name, POSIX_ACL_ACCESS_XATTR) == 0) ||
+                    (strcmp (name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
+                        send_fuse_err (this, finh, EOPNOTSUPP);
+                        GF_FREE (finh);
+                        return;
+                }
+        }
+
+        if (!priv->selinux) {
+                if (strncmp (name, "security.", 9) == 0) {
+                        send_fuse_err (this, finh, EOPNOTSUPP);
+                        GF_FREE (finh);
+                        return;
+                }
+        }
+
+        /* Check if the command is for changing the log
+           level of process or specific xlator */
+        ret = is_gf_log_command (this, name, value);
+        if (ret >= 0) {
+                send_fuse_err (this, finh, ret);
+                GF_FREE (finh);
+                return;
+        }
+
+        if (!strcmp ("inode-invalidate", name)) {
+                gf_log ("fuse", GF_LOG_TRACE,
+                        "got request to invalidate %"PRIu64, finh->nodeid);
+                send_fuse_err (this, finh, 0);
+                fuse_invalidate_entry (this, finh->nodeid);
+                GF_FREE (finh);
+                return;
+        }
+
+        if (!strcmp (GFID_XATTR_KEY, name)) {
+                send_fuse_err (this, finh, EPERM);
+                GF_FREE (finh);
+                return;
+        }
+
+        GET_STATE (this, finh, state);
+        state->size = fsi->size;
+
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+
+        state->xattr = get_new_dict ();
+        if (!state->xattr) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": SETXATTR %s/%"PRId64" (%s) (fuse_loc_fill() failed)", 
-                        req_callid (req),
-                        state->loc.path, (int64_t)ino, name);
+                        "%"PRIu64": SETXATTR dict allocation failed",
+                        finh->unique);
 
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+                send_fuse_err (this, finh, ENOMEM);
+                free_fuse_state (state);
                 return;
         }
 
-        state->dict = get_new_dict ();
+        ret = fuse_flip_xattr_ns (priv, name, &newkey);
+        if (ret) {
+                send_fuse_err (this, finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
 
-	dict_value = memdup (value, size);
-        dict_set (state->dict, (char *)name,
-                  data_from_dynptr ((void *)dict_value, size));
-        dict_ref (state->dict);
+        if (fsi->size > 0) {
+                dict_value = memdup (value, fsi->size);
+        } else {
+                gf_log (THIS->name, GF_LOG_ERROR, "value size zero");
+                dict_value = NULL;
+        }
+        dict_set (state->xattr, newkey,
+                  data_from_dynptr ((void *)dict_value, fsi->size));
+        dict_ref (state->xattr);
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": SETXATTR %s/%"PRId64" (%s)", req_callid (req),
-                state->loc.path, (int64_t)ino, name);
+        state->flags = fsi->flags;
+        state->name = newkey;
 
-        FUSE_FOP (state, fuse_err_cbk, GF_FOP_SETXATTR,
-                  setxattr, &state->loc, state->dict, flags);
+        fuse_resolve_and_resume (state, fuse_setxattr_resume);
 
         return;
 }
 
 
+static void
+send_fuse_xattr (xlator_t *this, fuse_in_header_t *finh, const char *value,
+                 size_t size, size_t expected)
+{
+        struct fuse_getxattr_out fgxo;
+
+        /* linux kernel limits the size of xattr value to 64k */
+        if (size > GLUSTERFS_XATTR_LEN_MAX)
+                send_fuse_err (this, finh, E2BIG);
+        else if (expected) {
+                /* if callback for getxattr and asks for value */
+                if (size > expected)
+                        /* reply would be bigger than
+                         * what was asked by kernel */
+                        send_fuse_err (this, finh, ERANGE);
+                else
+                        send_fuse_data (this, finh, (void *)value, size);
+        } else {
+                fgxo.size = size;
+                send_fuse_obj (this, finh, &fgxo);
+        }
+}
+
+/* filter out xattrs that need not be visible on the
+ * mount point. this is _specifically_ for geo-rep
+ * as of now, to prevent Rsync from crying out loud
+ * when it tries to setxattr() for selinux xattrs
+ */
 static int
-fuse_xattr_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                dict_t *dict)
-{
-	int need_to_free_dict = 0;
-        int32_t ret = op_ret;
-        char *value = "";
-        fuse_state_t *state = frame->root->state;
-        fuse_req_t req = state->req;
+fuse_filter_xattr(char *key)
+{
+        int need_filter = 0;
+        struct fuse_private *priv = THIS->private;
 
-#ifdef GF_DARWIN_HOST_OS
-	/* This is needed in MacFuse, where MacOSX Finder needs some specific 
-	 * keys to be supported from FS
-	 */
-	int32_t dummy_ret = 0;
-	if (state->name) {
-		if (!dict) {
-			dict = get_new_dict ();
-			need_to_free_dict = 1;
-		}
-		dummy_ret = gf_compat_getxattr (state->name, dict);
-		if (dummy_ret != -1)
-			ret = dummy_ret;
-	} else {
-		if (!dict) {
-			dict = get_new_dict ();
-			need_to_free_dict = 1;
-		}
-		dummy_ret = gf_compat_listxattr (ret, dict, state->size);
-		if (dummy_ret != -1)
-			ret = dummy_ret;
-	}
-#endif /* DARWIN */
-	
-        if (ret >= 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": %s() %s => %d", frame->root->unique,
+        if ((priv->client_pid == GF_CLIENT_PID_GSYNCD)
+            && fnmatch ("*.selinux*", key, FNM_PERIOD) == 0)
+                need_filter = 1;
+
+        return need_filter;
+}
+
+
+static int
+fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        int             need_to_free_dict = 0;
+        char           *value = "";
+        fuse_state_t   *state = NULL;
+        fuse_in_header_t *finh = NULL;
+        data_t         *value_data = NULL;
+        int             ret = -1;
+        int32_t         len = 0;
+        int32_t         len_next = 0;
+
+        state = frame->root->state;
+        finh  = state->finh;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+        if (op_ret >= 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": %s() %s => %d", frame->root->unique,
                         gf_fop_list[frame->root->op], state->loc.path, op_ret);
 
                 /* if successful */
                 if (state->name) {
                         /* if callback for getxattr */
-                        data_t *value_data = dict_get (dict, state->name);
+                        value_data = dict_get (dict, state->name);
                         if (value_data) {
+
                                 ret = value_data->len; /* Don't return the value for '\0' */
                                 value = value_data->data;
-        
-                                /* linux kernel limits the size of xattr value to 64k */
-                                if (ret > GLUSTERFS_XATTR_LEN_MAX) {
-                                        fuse_reply_err (req, E2BIG);
-                                } else if (state->size) {
-                                        /* if callback for getxattr and asks for value */
-                                        fuse_reply_buf (req, value, ret);
-                                } else {
-                                        /* if callback for getxattr and asks for value length only */
-                                        fuse_reply_xattr (req, ret);
-                                } /* if(ret >...)...else if...else */
-                        }  else if (!strcmp (state->name, "user.glusterfs-booster-volfile")) {
-				fuse_private_t *priv = this->private;
- 
-				if (!priv->volfile) {
-					int32_t fd = -1, ret = -1;
-					struct stat st;
-					char *file = NULL;
-					
-					memset (&st, 0, sizeof (st));
-					fd = fileno (this->ctx->specfp);
-					ret = fstat (fd, &st);
-					if (ret != 0) {
-						gf_log (this->name,
-							GF_LOG_ERROR,
-							"fstat on fd (%d) failed (%s)", fd, strerror (errno));
-						fuse_reply_err (req, ENODATA);
-					}
-					
-					priv->volfile_size = st.st_size;
-					file = priv->volfile = CALLOC (1, priv->volfile_size);
-					ret = lseek (fd, 0, SEEK_SET);
-					while ((ret = read (fd, file, GF_UNIT_KB)) > 0) {
-						file += ret;
-					}
-				}
-				
-				if (priv->volfile_size > GLUSTERFS_XATTR_LEN_MAX) {
-					fuse_reply_err (req, E2BIG);
-				} else if (state->size) {
-					/* if callback for getxattr and asks for value */
-					fuse_reply_buf (req, priv->volfile, priv->volfile_size);
-				} else {
-					/* if callback for getxattr and asks for value length only */
-					fuse_reply_xattr (req, priv->volfile_size);
-				} /* if(ret >...)...else if...else */
-			} else if (!strcmp (state->name, "user.glusterfs-booster-path")) {
-				if (state->size) {
-					fuse_reply_buf (req, state->loc.path, strlen (state->loc.path) + 1);
-				} else {
-					fuse_reply_xattr (req, strlen (state->loc.path) + 1);
-				}
-			} else {
-				fuse_reply_err (req, ENODATA);
-			} /* if(value_data)...else */
-		} else {
-			/* if callback for listxattr */
-                        int32_t len = 0;
-                        data_pair_t *trav = dict->members_list;
-                        while (trav) {
-                                len += strlen (trav->key) + 1;
-                                trav = trav->next;
-                        } /* while(trav) */
-                        value = alloca (len + 1);
-                        ERR_ABORT (value);
-                        len = 0;
-                        trav = dict->members_list;
-                        while (trav) {
-                                strcpy (value + len, trav->key);
-                                value[len + strlen(trav->key)] = '\0';
-                                len += strlen (trav->key) + 1;
-                                trav = trav->next;
-                        } /* while(trav) */
-                        if (state->size) {
-                                /* if callback for listxattr and asks for list of keys */
-                                fuse_reply_buf (req, value, len);
+
+                                send_fuse_xattr (this, finh, value, ret, state->size);
+                                /* if(ret >...)...else if...else */
                         } else {
-                                /* if callback for listxattr and asks for length of keys only */
-                                fuse_reply_xattr (req, len);
-                        } /* if(state->size)...else */
-		} /* if(state->name)...else */
+                                send_fuse_err (this, finh, ENODATA);
+                        } /* if(value_data)...else */
+                } else {
+                        /* if callback for listxattr */
+                        /* we need to invoke fuse_filter_xattr() twice. Once
+                         * while counting size and then while filling buffer
+                         */
+                        len = dict_keys_join (NULL, 0, dict, fuse_filter_xattr);
+                        if (len < 0)
+                                goto out;
+
+                        value = alloca (len + 1);
+                        if (!value)
+                                goto out;
+
+                        len_next = dict_keys_join (value, len, dict,
+                                                   fuse_filter_xattr);
+                        if (len_next != len)
+                                gf_log (THIS->name, GF_LOG_ERROR,
+                                        "sizes not equal %d != %d",
+                                        len, len_next);
+
+                        send_fuse_xattr (this, finh, value, len, state->size);
+                } /* if(state->name)...else */
         } else {
                 /* if failure - no need to check if listxattr or getxattr */
                 if (op_errno != ENODATA) {
-                        if (op_errno == ENOTSUP) 
-                        {
-                                gf_fuse_xattr_enotsup_log++;
-                                if (!(gf_fuse_xattr_enotsup_log % GF_UNIVERSAL_ANSWER))
-                                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                                "[ ERROR ] Extended attribute not supported by the backend storage");
-                        } 
-                        else 
-                        {
-                                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                        "%"PRId64": %s() %s => -1 (%s)",
-					frame->root->unique,
-                                        gf_fop_list[frame->root->op],
-					state->loc.path, strerror(op_errno));
+                        if (op_errno == ENOTSUP) {
+                                GF_LOG_OCCASIONALLY (gf_fuse_xattr_enotsup_log,
+                                                     "glusterfs-fuse",
+                                                     GF_LOG_ERROR,
+                                                     "extended attribute not "
+                                                     "supported by the backend "
+                                                     "storage");
+                        } else {
+                                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                        "%"PRIu64": %s(%s) %s => -1 (%s)",
+                                        frame->root->unique,
+                                        gf_fop_list[frame->root->op], state->name,
+                                        state->loc.path, strerror (op_errno));
                         }
                 } else {
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                                "%"PRId64": %s() %s => -1 (%s)",
-				frame->root->unique,
-                                gf_fop_list[frame->root->op], state->loc.path,
-				strerror(op_errno));
+                                "%"PRIu64": %s(%s) %s => -1 (%s)",
+                                frame->root->unique,
+                                gf_fop_list[frame->root->op], state->name,
+                                state->loc.path, strerror (op_errno));
                 } /* if(op_errno!= ENODATA)...else */
 
-                fuse_reply_err (req, op_errno);
+                send_fuse_err (this, finh, op_errno);
         } /* if(op_ret>=0)...else */
 
-	if (need_to_free_dict)
-		dict_unref (dict);
+out:
+        if (need_to_free_dict)
+                dict_unref (dict);
 
-        free_state (state);
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
 
+void
+fuse_getxattr_resume (fuse_state_t *state)
+{
+        char *value  = NULL;
+
+        if (!state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": GETXATTR %s/%"PRIu64" (%s) "
+                        "resolution failed",
+                        state->finh->unique,
+                        uuid_utoa (state->resolve.gfid),
+                        state->finh->nodeid, state->name);
+
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
+                return;
+        }
+
+#ifdef GF_TEST_FFOP
+        state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+#endif /* GF_TEST_FFOP */
+
+        if (state->name &&
+            (strcmp (state->name, VIRTUAL_GFID_XATTR_KEY) == 0)) {
+                /* send glusterfs gfid in binary form */
+
+                value = GF_CALLOC (16 + 1, sizeof(char),
+                                   gf_common_mt_char);
+                if (!value) {
+                        send_fuse_err (state->this, state->finh, ENOMEM);
+                        goto internal_out;
+                }
+                memcpy (value, state->loc.inode->gfid, 16);
+
+                send_fuse_xattr (THIS, state->finh, value, 16, state->size);
+                GF_FREE (value);
+        internal_out:
+                free_fuse_state (state);
+                return;
+        }
+
+        if (state->name &&
+            (strcmp (state->name, VIRTUAL_GFID_XATTR_KEY_STR) == 0)) {
+                /* transform binary gfid to canonical form */
+
+                value = GF_CALLOC (UUID_CANONICAL_FORM_LEN + 1, sizeof(char),
+                                   gf_common_mt_char);
+                if (!value) {
+                        send_fuse_err (state->this, state->finh, ENOMEM);
+                        goto internal_out1;
+                }
+                uuid_utoa_r (state->loc.inode->gfid, value);
+
+                send_fuse_xattr (THIS, state->finh, value,
+                                 UUID_CANONICAL_FORM_LEN, state->size);
+                GF_FREE (value);
+        internal_out1:
+                free_fuse_state (state);
+                return;
+        }
+
+
+        if (state->fd) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": GETXATTR %p/%"PRIu64" (%s)", state->finh->unique,
+                        state->fd, state->finh->nodeid, state->name);
+
+                FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_FGETXATTR,
+                          fgetxattr, state->fd, state->name, state->xdata);
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": GETXATTR %s/%"PRIu64" (%s)", state->finh->unique,
+                        state->loc.path, state->finh->nodeid, state->name);
+
+                FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
+                          getxattr, &state->loc, state->name, state->xdata);
+        }
+}
+
+
 static void
-fuse_getxattr (fuse_req_t req,
-               fuse_ino_t ino,
-               const char *name,
-               size_t size)
+fuse_getxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
+        struct fuse_getxattr_in *fgxi     = msg;
+        char                    *name     = (char *)(fgxi + 1);
+        fuse_state_t            *state    = NULL;
+        struct fuse_private     *priv     = NULL;
+        int                      rv       = 0;
+        int                     op_errno  = EINVAL;
+        char                    *newkey   = NULL;
 
-#ifdef DISABLE_POSIX_ACL
-	if (!strncmp (name, "system.", 7)) {
-		fuse_reply_err (req, ENODATA);
-		return;
-	}
+        priv = this->private;
+        GET_STATE (this, finh, state);
+
+#ifdef GF_DARWIN_HOST_OS
+        if (fgxi->position) {
+                /* position can be used only for
+                 * resource fork queries which we
+                 * don't support anyway... so handling
+                 * it separately is just sort of a
+                 * matter of aesthetics, not strictly
+                 * necessary.
+                 */
+
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": GETXATTR %s/%"PRIu64" (%s):"
+                        "refusing positioned getxattr",
+                        finh->unique, state->loc.path, finh->nodeid, name);
+                op_errno = EINVAL;
+                goto err;
+        }
 #endif
 
-        state = state_from_req (req);
-        state->size = size;
-        state->name = strdup (name);
+        if (!priv->acl) {
+                if ((strcmp (name, POSIX_ACL_ACCESS_XATTR) == 0) ||
+                    (strcmp (name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
+                        op_errno = ENOTSUP;
+                        goto err;
+                }
+        }
 
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": GETXATTR %s/%"PRId64" (%s) (fuse_loc_fill() failed)", 
-                        req_callid (req), state->loc.path, (int64_t)ino, name);
+        if (!priv->selinux) {
+                if (strncmp (name, "security.", 9) == 0) {
+                        op_errno = ENODATA;
+                        goto err;
+                }
+        }
 
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
-                return;
+        GET_STATE (this, finh, state);
+
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+
+        rv = fuse_flip_xattr_ns (priv, name, &newkey);
+        if (rv) {
+                op_errno = ENOMEM;
+                goto err;
         }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": GETXATTR %s/%"PRId64" (%s)", req_callid (req),
-                state->loc.path, (int64_t)ino, name);
+        state->size = fgxi->size;
+        state->name = newkey;
 
-        FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
-                  getxattr, &state->loc, name);
+        fuse_resolve_and_resume (state, fuse_getxattr_resume);
 
         return;
+ err:
+        send_fuse_err (this, finh, op_errno);
+        free_fuse_state (state);
+        return;
+}
+
+
+void
+fuse_listxattr_resume (fuse_state_t *state)
+{
+        if (!state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": LISTXATTR %s/%"PRIu64
+                        "resolution failed", state->finh->unique,
+                        uuid_utoa (state->resolve.gfid), state->finh->nodeid);
+
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
+                return;
+        }
+
+#ifdef GF_TEST_FFOP
+        state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+#endif /* GF_TEST_FFOP */
+
+        if (state->fd) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": LISTXATTR %p/%"PRIu64, state->finh->unique,
+                        state->fd, state->finh->nodeid);
+
+                FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_FGETXATTR,
+                          fgetxattr, state->fd, NULL, state->xdata);
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": LISTXATTR %s/%"PRIu64, state->finh->unique,
+                        state->loc.path, state->finh->nodeid);
+
+                FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
+                          getxattr, &state->loc, NULL, state->xdata);
+        }
 }
 
 
 static void
-fuse_listxattr (fuse_req_t req,
-                fuse_ino_t ino,
-                size_t size)
+fuse_listxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
+        struct fuse_getxattr_in *fgxi = msg;
+        fuse_state_t *state = NULL;
+
+        GET_STATE (this, finh, state);
+
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+
+        state->size = fgxi->size;
+
+        fuse_resolve_and_resume (state, fuse_listxattr_resume);
+
+        return;
+}
 
-        state = state_from_req (req);
-        state->size = size;
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": LISTXATTR %s/%"PRId64" (fuse_loc_fill() failed)", 
-                        req_callid (req), state->loc.path, (int64_t)ino);
 
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+void
+fuse_removexattr_resume (fuse_state_t *state)
+{
+         if (!state->loc.inode) {
+                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+                        "%"PRIu64": REMOVEXATTR %s/%"PRIu64" (%s) "
+                        "resolution failed",
+                        state->finh->unique, uuid_utoa (state->resolve.gfid),
+                        state->finh->nodeid, state->name);
+
+                send_fuse_err (state->this, state->finh, ENOENT);
+                free_fuse_state (state);
                 return;
         }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": LISTXATTR %s/%"PRId64, req_callid (req),
-                state->loc.path, (int64_t)ino);
+#ifdef GF_TEST_FFOP
+        state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+#endif /* GF_TEST_FFOP */
 
-        FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
-                  getxattr, &state->loc, NULL);
+        if (state->fd) {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": REMOVEXATTR %p/%"PRIu64" (%s)", state->finh->unique,
+                        state->fd, state->finh->nodeid, state->name);
 
-        return;
+                FUSE_FOP (state, fuse_err_cbk, GF_FOP_FREMOVEXATTR,
+                          fremovexattr, state->fd, state->name, state->xdata);
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": REMOVEXATTR %s/%"PRIu64" (%s)", state->finh->unique,
+                        state->loc.path, state->finh->nodeid, state->name);
+
+                FUSE_FOP (state, fuse_err_cbk, GF_FOP_REMOVEXATTR,
+                          removexattr, &state->loc, state->name, state->xdata);
+        }
 }
 
 
 static void
-fuse_removexattr (fuse_req_t req,
-                  fuse_ino_t ino,
-                  const char *name)
-
+fuse_removexattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	int32_t ret = -1;
+        char *name = msg;
 
-        state = state_from_req (req);
-        ret = fuse_loc_fill (&state->loc, state, ino, 0, NULL);
-        if ((state->loc.inode == NULL) ||
-	    (ret < 0)) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRId64": REMOVEXATTR %s/%"PRId64" (%s) (fuse_loc_fill() failed)",
-                        req_callid (req), state->loc.path, (int64_t)ino, name);
+        fuse_state_t *state = NULL;
+        fuse_private_t *priv = NULL;
+        int32_t       ret = -1;
+        char *newkey = NULL;
 
-                fuse_reply_err (req, EINVAL);
-		free_state (state);
+        if (!strcmp (GFID_XATTR_KEY, name)) {
+                send_fuse_err (this, finh, EPERM);
+                GF_FREE (finh);
                 return;
         }
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": REMOVEXATTR %s/%"PRId64" (%s)", req_callid (req),
-                state->loc.path, (int64_t)ino, name);
+        priv = this->private;
 
-        FUSE_FOP (state, fuse_err_cbk, GF_FOP_REMOVEXATTR,
-                  removexattr, &state->loc, name);
+        GET_STATE (this, finh, state);
 
+        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+
+        ret = fuse_flip_xattr_ns (priv, name, &newkey);
+        if (ret) {
+                send_fuse_err (this, finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
+        state->name = newkey;
+
+        fuse_resolve_and_resume (state, fuse_removexattr_resume);
         return;
 }
 
@@ -2290,513 +3515,1891 @@ fuse_removexattr (fuse_req_t req,
 static int gf_fuse_lk_enosys_log;
 
 static int
-fuse_getlk_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                struct flock *lock)
+fuse_getlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
+        fuse_state_t *state = NULL;
+
+        state = frame->root->state;
+        struct fuse_lk_out flo = {{0, }, };
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
         if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": ERR => 0", frame->root->unique);
-                fuse_reply_lock (state->req, lock);
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": ERR => 0", frame->root->unique);
+                flo.lk.type = lock->l_type;
+                flo.lk.pid  = lock->l_pid;
+                if (lock->l_type == F_UNLCK)
+                        flo.lk.start = flo.lk.end = 0;
+                else {
+                        flo.lk.start = lock->l_start;
+                        flo.lk.end = lock->l_len ?
+                                     (lock->l_start + lock->l_len - 1) :
+                                     OFFSET_MAX;
+                }
+                send_fuse_obj (this, state->finh, &flo);
         } else {
                 if (op_errno == ENOSYS) {
                         gf_fuse_lk_enosys_log++;
                         if (!(gf_fuse_lk_enosys_log % GF_UNIVERSAL_ANSWER)) {
-				gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-					"[ ERROR ] loading 'features/posix-locks' on server side may help your application");
+                                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                        "GETLK not supported. loading "
+                                        "'features/posix-locks' on server side "
+                                        "will add GETLK support.");
                         }
                 } else {
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "%"PRId64": ERR => -1 (%s)",
-				frame->root->unique, strerror (op_errno));
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "%"PRIu64": ERR => -1 (%s)",
+                                frame->root->unique, strerror (op_errno));
                 }
-                fuse_reply_err (state->req, op_errno);
+                send_fuse_err (this, state->finh, op_errno);
         }
-        
-        free_state (state);
+
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
 
+void
+fuse_getlk_resume (fuse_state_t *state)
+{
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": GETLK %p", state->finh->unique, state->fd);
+
+        FUSE_FOP (state, fuse_getlk_cbk, GF_FOP_LK,
+                  lk, state->fd, F_GETLK, &state->lk_lock, state->xdata);
+}
+
+
 static void
-fuse_getlk (fuse_req_t req,
-            fuse_ino_t ino,
-            struct fuse_file_info *fi,
-            struct flock *lock)
+fuse_getlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-	fd_t *fd = NULL;
-	
-	fd = FI_TO_FD (fi);
-        state = state_from_req (req);
-        state->req = req;
-	state->fd = fd;
+        struct fuse_lk_in *fli = msg;
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": GETLK %p", req_callid (req), fd);
+        fuse_state_t *state = NULL;
+        fd_t         *fd = NULL;
 
-        FUSE_FOP (state, fuse_getlk_cbk, GF_FOP_LK,
-                  lk, fd, F_GETLK, lock);
+        fd = FH_TO_FD (fli->fh);
+        GET_STATE (this, finh, state);
+        state->fd = fd;
+
+        fuse_resolve_fd_init (state, &state->resolve, fd);
+
+        convert_fuse_file_lock (&fli->lk, &state->lk_lock,
+                                fli->owner);
+
+        state->lk_owner = fli->owner;
+
+        fuse_resolve_and_resume (state, fuse_getlk_resume);
 
         return;
 }
 
 
 static int
-fuse_setlk_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                struct flock *lock)
+fuse_setlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
+        uint32_t      op    = 0;
+        fuse_state_t *state = NULL;
+
+        state = frame->root->state;
+        op    = state->finh->opcode;
+
+        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
         if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRId64": ERR => 0", frame->root->unique);
-                fuse_reply_err (state->req, 0);
+                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                        "%"PRIu64": ERR => 0", frame->root->unique);
+                fd_lk_insert_and_merge (state->fd,
+                                        (op == FUSE_SETLK) ? F_SETLK : F_SETLKW,
+                                        &state->lk_lock);
+
+                send_fuse_err (this, state->finh, 0);
         } else {
                 if (op_errno == ENOSYS) {
                         gf_fuse_lk_enosys_log++;
                         if (!(gf_fuse_lk_enosys_log % GF_UNIVERSAL_ANSWER)) {
-				gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                        "[ ERROR ] loading 'features/posix-locks' on server side may help your application");
+                                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                        "SETLK not supported. loading "
+                                        "'features/posix-locks' on server side "
+                                        "will add SETLK support.");
                         }
+                } else if (op_errno == EAGAIN) {
+                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+                                "Returning EAGAIN Flock: "
+                                "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
+                                (unsigned long long) state->lk_lock.l_start,
+                                (unsigned long long) state->lk_lock.l_len,
+                                (unsigned long long) state->lk_lock.l_pid,
+                                lkowner_utoa (&frame->root->lk_owner));
                 } else  {
-                        gf_log ("glusterfs-fuse",
-				(op_errno == EAGAIN) ? GF_LOG_DEBUG : GF_LOG_ERROR,
-                                "%"PRId64": ERR => -1 (%s)",
-				frame->root->unique, strerror (op_errno));
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "%"PRIu64": ERR => -1 (%s)",
+                                frame->root->unique, strerror (op_errno));
                 }
 
-                fuse_reply_err (state->req, op_errno);
+                send_fuse_err (this, state->finh, op_errno);
         }
-        
-        free_state (state);
+
+        free_fuse_state (state);
         STACK_DESTROY (frame->root);
 
         return 0;
 }
 
 
+void
+fuse_setlk_resume (fuse_state_t *state)
+{
+        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
+                "%"PRIu64": SETLK%s %p", state->finh->unique,
+                state->finh->opcode == FUSE_SETLK ? "" : "W", state->fd);
+
+        FUSE_FOP (state, fuse_setlk_cbk, GF_FOP_LK, lk, state->fd,
+                  state->finh->opcode == FUSE_SETLK ? F_SETLK : F_SETLKW,
+                  &state->lk_lock, state->xdata);
+}
+
+
 static void
-fuse_setlk (fuse_req_t req,
-            fuse_ino_t ino,
-            struct fuse_file_info *fi,
-            struct flock *lock,
-            int sleep)
+fuse_setlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        fuse_state_t *state;
-        fd_t *fd = NULL;
-        
-        fd = FI_TO_FD (fi);
-        state = state_from_req (req);
-        state->req = req;
+        struct fuse_lk_in *fli = msg;
+
+        fuse_state_t *state = NULL;
+        fd_t         *fd = NULL;
+
+        fd = FH_TO_FD (fli->fh);
+        GET_STATE (this, finh, state);
+        state->finh = finh;
         state->fd = fd;
 
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                "%"PRId64": SETLK %p (sleep=%d)", req_callid (req), fd,
-                sleep);
+        fuse_resolve_fd_init (state, &state->resolve, fd);
 
-        FUSE_FOP (state, fuse_setlk_cbk, GF_FOP_LK,
-                  lk, fd, (sleep ? F_SETLKW : F_SETLK), lock);
+        convert_fuse_file_lock (&fli->lk, &state->lk_lock,
+                                fli->owner);
+
+        state->lk_owner = fli->owner;
+
+        fuse_resolve_and_resume (state, fuse_setlk_resume);
 
         return;
 }
 
 
-static void 
-fuse_init (void *data, struct fuse_conn_info *conn)
+static void *
+notify_kernel_loop (void *data)
 {
-        xlator_t *this_xl = NULL;
-        
-        if (data == NULL) {
-                return ;
+        xlator_t               *this = NULL;
+        fuse_private_t         *priv = NULL;
+        struct fuse_out_header *fouh = NULL;
+        int                     rv   = 0;
+
+        char inval_buf[INVAL_BUF_SIZE] = {0,};
+
+        this = data;
+        priv = this->private;
+
+        for (;;) {
+                rv = read (priv->revchan_in, inval_buf, sizeof (*fouh));
+                if (rv != sizeof (*fouh))
+                        break;
+                fouh = (struct fuse_out_header *)inval_buf;
+                rv = read (priv->revchan_in, inval_buf + sizeof (*fouh),
+                           fouh->len - sizeof (*fouh));
+                if (rv != fouh->len - sizeof (*fouh))
+                        break;
+                rv = write (priv->fd, inval_buf, fouh->len);
+                if (rv != fouh->len && !(rv == -1 && errno == ENOENT))
+                        break;
         }
-        
-        this_xl = data;
-        
-        this_xl->itable = inode_table_new (0, this_xl);
-        
-        return ;
+
+        close (priv->revchan_in);
+        close (priv->revchan_out);
+
+        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                "kernel notifier loop terminated");
+
+        return NULL;
 }
 
+
 static void
-fuse_destroy (void *data)
-{
-
-}
-
-static struct fuse_lowlevel_ops fuse_ops = {
-        .init         = fuse_init,
-        .destroy      = fuse_destroy,
-        .lookup       = fuse_lookup,
-        .forget       = fuse_forget,
-        .getattr      = fuse_getattr,
-        .setattr      = fuse_setattr,
-        .opendir      = fuse_opendir,
-        .readdir      = fuse_readdir,
-        .releasedir   = fuse_releasedir,
-        .access       = fuse_access,
-        .readlink     = fuse_readlink,
-        .mknod        = fuse_mknod,
-        .mkdir        = fuse_mkdir,
-        .unlink       = fuse_unlink,
-        .rmdir        = fuse_rmdir,
-        .symlink      = fuse_symlink,
-        .rename       = fuse_rename,
-        .link         = fuse_link,
-        .create       = fuse_create,
-        .open         = fuse_open,
-        .read         = fuse_readv,
-        .write        = fuse_write,
-        .flush        = fuse_flush,
-        .release      = fuse_release,
-        .fsync        = fuse_fsync,
-        .fsyncdir     = fuse_fsyncdir,
-        .statfs       = fuse_statfs,
-        .setxattr     = fuse_setxattr,
-        .getxattr     = fuse_getxattr,
-        .listxattr    = fuse_listxattr,
-        .removexattr  = fuse_removexattr,
-        .getlk        = fuse_getlk,
-        .setlk        = fuse_setlk
+fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        struct fuse_init_in  *fini      = msg;
+        struct fuse_init_out  fino      = {0,};
+        fuse_private_t       *priv      = NULL;
+        int                   ret       = 0;
+        int                   pfd[2]    = {0,};
+        pthread_t             messenger;
+
+        priv = this->private;
+
+        if (priv->init_recvd) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "got INIT after first message");
+
+                close (priv->fd);
+                goto out;
+        }
+
+        priv->init_recvd = 1;
+
+        if (fini->major != FUSE_KERNEL_VERSION) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "unsupported FUSE protocol version %d.%d",
+                        fini->major, fini->minor);
+
+                close (priv->fd);
+                goto out;
+        }
+        priv->proto_minor = fini->minor;
+
+        fino.major = FUSE_KERNEL_VERSION;
+        fino.minor = FUSE_KERNEL_MINOR_VERSION;
+        fino.max_readahead = 1 << 17;
+        fino.max_write = 1 << 17;
+        fino.flags = FUSE_ASYNC_READ | FUSE_POSIX_LOCKS;
+#if FUSE_KERNEL_MINOR_VERSION >= 17
+	if (fini->minor >= 17)
+		fino.flags |= FUSE_FLOCK_LOCKS;
+#endif
+#if FUSE_KERNEL_MINOR_VERSION >= 12
+        if (fini->minor >= 12) {
+            /* let fuse leave the umask processing to us, so that it does not
+             * break extended POSIX ACL defaults on server */
+            fino.flags |= FUSE_DONT_MASK;
+        }
+#endif
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+        if (fini->minor >= 6 /* fuse_init_in has flags */ &&
+            fini->flags & FUSE_BIG_WRITES) {
+                /* no need for direct I/O mode by default if big writes are supported */
+                if (priv->direct_io_mode == 2)
+                        priv->direct_io_mode = 0;
+                fino.flags |= FUSE_BIG_WRITES;
+        }
+
+        /* Used for 'reverse invalidation of inode' */
+        if (fini->minor >= 12) {
+                if (pipe(pfd) == -1) {
+                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                "cannot create pipe pair (%s)",
+                                strerror(errno));
+
+                        close (priv->fd);
+                        goto out;
+                }
+                priv->revchan_in  = pfd[0];
+                priv->revchan_out = pfd[1];
+                ret = gf_thread_create (&messenger, NULL, notify_kernel_loop,
+					this);
+                if (ret != 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                "failed to start messenger daemon (%s)",
+                                strerror(errno));
+
+                        close (priv->fd);
+                        goto out;
+                }
+                priv->reverse_fuse_thread_started = _gf_true;
+        } else {
+                /*
+                 * FUSE minor < 12 does not implement invalidate notifications.
+                 * This mechanism is required for fopen-keep-cache to operate
+                 * correctly. Disable and warn the user.
+                 */
+                if (priv->fopen_keep_cache) {
+                        gf_log("glusterfs-fuse", GF_LOG_WARNING, "FUSE version "
+                                "%d.%d does not support inval notifications. "
+                                "fopen-keep-cache disabled.", fini->major,
+                                fini->minor);
+                        priv->fopen_keep_cache = 0;
+                }
+        }
+
+        if (fini->minor >= 13) {
+                fino.max_background = priv->background_qlen;
+                fino.congestion_threshold = priv->congestion_threshold;
+        }
+        if (fini->minor < 9)
+                *priv->msg0_len_p = sizeof(*finh) + FUSE_COMPAT_WRITE_IN_SIZE;
+#endif
+        if (priv->use_readdirp) {
+                if (fini->flags & FUSE_DO_READDIRPLUS)
+                        fino.flags |= FUSE_DO_READDIRPLUS;
+        }
+
+	if (priv->fopen_keep_cache == 2) {
+		/* If user did not explicitly set --fopen-keep-cache[=off],
+		   then check if kernel support FUSE_AUTO_INVAL_DATA and ...
+		*/
+		if (fini->flags & FUSE_AUTO_INVAL_DATA) {
+			/* ... enable fopen_keep_cache mode if supported.
+			*/
+			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "Detected "
+				"support for FUSE_AUTO_INVAL_DATA. Enabling "
+				"fopen_keep_cache automatically.");
+			fino.flags |= FUSE_AUTO_INVAL_DATA;
+			priv->fopen_keep_cache = 1;
+		} else {
+			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "No support "
+				"for FUSE_AUTO_INVAL_DATA. Disabling "
+				"fopen_keep_cache.");
+			/* ... else disable. */
+			priv->fopen_keep_cache = 0;
+		}
+	} else if (priv->fopen_keep_cache == 1) {
+		/* If user explicitly set --fopen-keep-cache[=on],
+		   then enable FUSE_AUTO_INVAL_DATA if possible.
+		*/
+		if (fini->flags & FUSE_AUTO_INVAL_DATA) {
+			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "fopen_keep_cache "
+				"is explicitly set. Enabling FUSE_AUTO_INVAL_DATA");
+			fino.flags |= FUSE_AUTO_INVAL_DATA;
+		} else {
+			gf_log ("glusterfs-fuse", GF_LOG_WARNING, "fopen_keep_cache "
+				"is explicitly set. Support for "
+				"FUSE_AUTO_INVAL_DATA is missing");
+		}
+	}
+
+	if (fini->flags & FUSE_ASYNC_DIO)
+		fino.flags |= FUSE_ASYNC_DIO;
+
+        ret = send_fuse_obj (this, finh, &fino);
+        if (ret == 0)
+                gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                        "FUSE inited with protocol versions:"
+                        " glusterfs %d.%d kernel %d.%d",
+                        FUSE_KERNEL_VERSION, FUSE_KERNEL_MINOR_VERSION,
+                        fini->major, fini->minor);
+        else {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "FUSE init failed (%s)", strerror (ret));
+
+                close (priv->fd);
+        }
+
+ out:
+        GF_FREE (finh);
+}
+
+
+static void
+fuse_enosys (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        send_fuse_err (this, finh, ENOSYS);
+
+        GF_FREE (finh);
+}
+
+
+static void
+fuse_destroy (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        send_fuse_err (this, finh, 0);
+
+        GF_FREE (finh);
+}
+
+
+
+struct fuse_first_lookup {
+        pthread_mutex_t  mutex;
+        pthread_cond_t   cond;
+        char             fin;
 };
 
+int
+fuse_first_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       inode_t *inode, struct iatt *buf, dict_t *xattr,
+                       struct iatt *postparent)
+{
+        struct fuse_first_lookup *stub = NULL;
+
+        stub = frame->local;
+
+        if (op_ret == 0) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "first lookup on root succeeded.");
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "first lookup on root failed.");
+        }
+
+        pthread_mutex_lock (&stub->mutex);
+        {
+                stub->fin = 1;
+                pthread_cond_broadcast (&stub->cond);
+        }
+        pthread_mutex_unlock (&stub->mutex);
+
+        return 0;
+}
+
+
+int
+fuse_first_lookup (xlator_t *this)
+{
+        fuse_private_t            *priv = NULL;
+        loc_t                      loc = {0, };
+        call_frame_t              *frame = NULL;
+        xlator_t                  *xl = NULL;
+        dict_t                    *dict = NULL;
+        struct fuse_first_lookup   stub;
+        uuid_t                     gfid;
+        int                        ret;
+
+        priv = this->private;
+
+        loc.path = "/";
+        loc.name = "";
+        loc.inode = fuse_ino_to_inode (1, this);
+        uuid_copy (loc.gfid, loc.inode->gfid);
+        loc.parent = NULL;
+
+        dict = dict_new ();
+        frame = create_frame (this, this->ctx->pool);
+        frame->root->type = GF_OP_TYPE_FOP;
+
+        xl = priv->active_subvol;
+
+        pthread_mutex_init (&stub.mutex, NULL);
+        pthread_cond_init (&stub.cond, NULL);
+        stub.fin = 0;
+
+        frame->local = &stub;
+
+        memset (gfid, 0, 16);
+        gfid[15] = 1;
+        ret = dict_set_static_bin (dict, "gfid-req", gfid, 16);
+        if (ret)
+                gf_log (xl->name, GF_LOG_ERROR, "failed to set 'gfid-req'");
+
+        STACK_WIND (frame, fuse_first_lookup_cbk, xl, xl->fops->lookup,
+                    &loc, dict);
+        dict_unref (dict);
+
+        pthread_mutex_lock (&stub.mutex);
+        {
+                while (!stub.fin) {
+                        pthread_cond_wait (&stub.cond, &stub.mutex);
+                }
+        }
+        pthread_mutex_unlock (&stub.mutex);
+
+        pthread_mutex_destroy (&stub.mutex);
+        pthread_cond_destroy (&stub.cond);
+
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        return 0;
+}
+
+
+int
+fuse_nameless_lookup (xlator_t *xl, uuid_t gfid, loc_t *loc)
+{
+        int          ret          = -1;
+        dict_t      *xattr_req    = NULL;
+        struct iatt  iatt         = {0, };
+        inode_t     *linked_inode = NULL;
+
+        if ((loc == NULL) || (xl == NULL)) {
+                goto out;
+        }
+
+        if (loc->inode == NULL) {
+                loc->inode = inode_new (xl->itable);
+                if (loc->inode == NULL) {
+                        goto out;
+                }
+        }
+
+        uuid_copy (loc->gfid, gfid);
+
+        xattr_req = dict_new ();
+        if (xattr_req == NULL) {
+                goto out;
+        }
+
+        ret = syncop_lookup (xl, loc, xattr_req, &iatt, NULL, NULL);
+        if (ret < 0) {
+                goto out;
+        }
+
+        linked_inode = inode_link (loc->inode, NULL, NULL, &iatt);
+        inode_unref (loc->inode);
+        loc->inode = linked_inode;
+
+        ret = 0;
+out:
+        if (xattr_req != NULL) {
+                dict_unref (xattr_req);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_migrate_fd_open (xlator_t *this, fd_t *basefd, fd_t *oldfd,
+                      xlator_t *old_subvol, xlator_t *new_subvol)
+{
+        loc_t          loc       = {0, };
+        fd_t          *newfd     = NULL, *old_activefd = NULL;
+        fuse_fd_ctx_t *basefd_ctx = NULL;
+        fuse_fd_ctx_t *newfd_ctx = NULL;
+        int            ret       = 0, flags = 0;
+
+        ret = inode_path (basefd->inode, NULL, (char **)&loc.path);
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "cannot construct path of gfid (%s) failed"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                        uuid_utoa (basefd->inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+        uuid_copy (loc.gfid, basefd->inode->gfid);
+
+        loc.inode = inode_find (new_subvol->itable, basefd->inode->gfid);
+
+        if (loc.inode == NULL) {
+                ret = fuse_nameless_lookup (new_subvol, basefd->inode->gfid,
+                                            &loc);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "name-less lookup of gfid (%s) failed (%s)"
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                uuid_utoa (basefd->inode->gfid),
+                                strerror (errno),
+                                old_subvol->name, old_subvol->graph->id,
+                                new_subvol->name, new_subvol->graph->id);
+                        goto out;
+                }
+
+        }
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        newfd = fd_create (loc.inode, basefd->pid);
+        if (newfd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "cannot create new fd, hence not migrating basefd "
+                        "(ptr:%p inode-gfid:%s) "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (loc.inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+        newfd->flags = basefd->flags;
+	if (newfd->lk_ctx)
+		fd_lk_ctx_unref (newfd->lk_ctx);
+
+        newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
+
+        newfd_ctx = fuse_fd_ctx_check_n_create (this, newfd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", newfd_ctx, out);
+
+        if (IA_ISDIR (basefd->inode->ia_type)) {
+                ret = syncop_opendir (new_subvol, &loc, newfd);
+        } else {
+                flags = basefd->flags & ~(O_CREAT | O_EXCL | O_TRUNC);
+                ret = syncop_open (new_subvol, &loc, flags, newfd);
+        }
+
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "open on basefd (ptr:%p inode-gfid:%s) failed (%s)"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid), strerror (errno),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+        fd_bind (newfd);
+
+        LOCK (&basefd->lock);
+        {
+                if (basefd_ctx->activefd != NULL) {
+                        old_activefd = basefd_ctx->activefd;
+                }
+
+                basefd_ctx->activefd = newfd;
+        }
+        UNLOCK (&basefd->lock);
+
+        if (old_activefd != NULL) {
+                fd_unref (old_activefd);
+        }
+
+        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                "migrated basefd (%p) to newfd (%p) (inode-gfid:%s)"
+                "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd, newfd,
+                uuid_utoa (basefd->inode->gfid),
+		old_subvol->name, old_subvol->graph->id,
+                new_subvol->name, new_subvol->graph->id);
+
+        ret = 0;
+
+out:
+        loc_wipe (&loc);
+
+        return ret;
+}
+
+int
+fuse_migrate_locks (xlator_t *this, fd_t *basefd, fd_t *oldfd,
+                    xlator_t *old_subvol, xlator_t *new_subvol)
+{
+        int	 ret      = -1;
+        dict_t	*lockinfo = NULL;
+        void    *ptr      = NULL;
+	fd_t    *newfd    = NULL;
+        fuse_fd_ctx_t *basefd_ctx         = NULL;
+
+
+	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
+		return 0;
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        LOCK (&basefd->lock);
+        {
+                newfd = fd_ref (basefd_ctx->activefd);
+        }
+        UNLOCK (&basefd->lock);
+
+        ret = syncop_fgetxattr (old_subvol, oldfd, &lockinfo,
+                                GF_XATTR_LOCKINFO_KEY);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+			"getting lockinfo failed while migrating locks"
+			"(oldfd:%p newfd:%p inode-gfid:%s)"
+			"(old-subvol:%s-%d new-subvol:%s-%d)",
+			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
+			old_subvol->name, old_subvol->graph->id,
+			new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+        ret = dict_get_ptr (lockinfo, GF_XATTR_LOCKINFO_KEY, &ptr);
+        if (ptr == NULL) {
+                ret = 0;
+                gf_log (this->name, GF_LOG_INFO,
+                        "No lockinfo present on any of the bricks "
+			"(oldfd: %p newfd:%p inode-gfid:%s) "
+                        "(old-subvol:%s-%d new-subvol:%s-%d)",
+			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
+			old_subvol->name, old_subvol->graph->id,
+			new_subvol->name, new_subvol->graph->id);
+
+                goto out;
+        }
+
+        ret = syncop_fsetxattr (new_subvol, newfd, lockinfo, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+			"migrating locks failed (oldfd:%p newfd:%p "
+			"inode-gfid:%s) (old-subvol:%s-%d new-subvol:%s-%d)",
+			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
+			old_subvol->name, old_subvol->graph->id,
+			new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+out:
+	if (newfd)
+		fd_unref (newfd);
+
+        if (lockinfo != NULL) {
+                dict_unref (lockinfo);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_migrate_fd (xlator_t *this, fd_t *basefd, xlator_t *old_subvol,
+                 xlator_t *new_subvol)
+{
+        int            ret                = -1;
+        char           create_in_progress = 0;
+        fuse_fd_ctx_t *basefd_ctx         = NULL;
+        fd_t          *oldfd              = NULL;
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        LOCK (&basefd->lock);
+        {
+                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd
+                        : basefd;
+                fd_ref (oldfd);
+        }
+        UNLOCK (&basefd->lock);
+
+        LOCK (&oldfd->inode->lock);
+        {
+                if (uuid_is_null (oldfd->inode->gfid)) {
+                        create_in_progress = 1;
+                } else {
+                        create_in_progress = 0;
+                }
+        }
+        UNLOCK (&oldfd->inode->lock);
+
+        if (create_in_progress) {
+                gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                        "create call on fd (%p) is in progress "
+                        "(basefd-ptr:%p basefd-inode.gfid:%s), "
+                        "hence deferring migration till application does an "
+                        "fd based operation on this fd"
+                        "(old-subvolume:%s-%d, new-subvolume:%s-%d)",
+                        oldfd, basefd, uuid_utoa (basefd->inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+
+                ret = 0;
+                goto out;
+        }
+
+        if (oldfd->inode->table->xl == old_subvol) {
+                ret = syncop_fsync (old_subvol, oldfd, 0);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "syncop_fsync failed (%s) on fd (%p)"
+                                "(basefd:%p basefd-inode.gfid:%s) "
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                strerror (errno), oldfd, basefd,
+                                uuid_utoa (basefd->inode->gfid),
+                                old_subvol->name, old_subvol->graph->id,
+                                new_subvol->name, new_subvol->graph->id);
+                }
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "basefd (ptr:%p inode-gfid:%s) was not "
+                        "migrated during previous graph switch"
+                        "(old-subvolume:%s-%d new-subvolume: %s-%d)", basefd,
+                        basefd->inode->gfid,
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+        }
+
+        ret = fuse_migrate_fd_open (this, basefd, oldfd, old_subvol,
+                                    new_subvol);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "open corresponding to "
+                        "basefd (ptr:%p inode-gfid:%s) in new graph failed "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid), old_subvol->name,
+                        old_subvol->graph->id, new_subvol->name,
+                        new_subvol->graph->id);
+                goto out;
+        }
+
+        ret = fuse_migrate_locks (this, basefd, oldfd, old_subvol,
+                                  new_subvol);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "migrating locks from old-subvolume (%s-%d) to "
+                        "new-subvolume (%s-%d) failed (inode-gfid:%s oldfd:%p "
+                        "basefd:%p)", old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id,
+                        uuid_utoa (basefd->inode->gfid), oldfd, basefd);
+
+        }
+out:
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "migration of basefd "
+                        "(ptr:%p inode-gfid:%s) failed"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        oldfd ? uuid_utoa (oldfd->inode->gfid) : NULL,
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+        }
+
+        fd_unref (oldfd);
+
+        return ret;
+}
+
+
+int
+fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
+                        xlator_t *new_subvol)
+{
+        fuse_private_t *priv      = NULL;
+        fdentry_t      *fdentries = NULL;
+        uint32_t        count     = 0;
+        fdtable_t      *fdtable   = NULL;
+        int             i         = 0;
+        fd_t           *fd        = NULL;
+        int32_t         ret       = 0;
+        fuse_fd_ctx_t  *fdctx     = NULL;
+
+        priv = this->private;
+
+        fdtable = priv->fdtable;
+
+        fdentries = gf_fd_fdtable_copy_all_fds (fdtable, &count);
+        if (fdentries != NULL) {
+                for (i = 0; i < count; i++) {
+                        fd = fdentries[i].fd;
+                        if (fd == NULL)
+                                continue;
+
+                        ret = fuse_migrate_fd (this, fd, old_subvol,
+                                               new_subvol);
+
+                        fdctx = fuse_fd_ctx_get (this, fd);
+                        if (fdctx) {
+                                LOCK (&fd->lock);
+                                {
+                                        if (ret < 0) {
+                                                fdctx->migration_failed = 1;
+                                        } else {
+                                                fdctx->migration_failed = 0;
+                                        }
+                                }
+                                UNLOCK (&fd->lock);
+                        }
+                }
+
+                for (i = 0; i < count ; i++) {
+                        fd = fdentries[i].fd;
+                        if (fd)
+                                fd_unref (fd);
+                }
+
+                GF_FREE (fdentries);
+        }
+
+        return 0;
+}
+
+
+static int
+fuse_handle_blocked_locks (xlator_t *this, xlator_t *old_subvol,
+                           xlator_t *new_subvol)
+{
+        return 0;
+}
+
+
+static int
+fuse_graph_switch_task (void *data)
+{
+        fuse_graph_switch_args_t *args = NULL;
+
+        args = data;
+        if (args == NULL) {
+                goto out;
+        }
+
+        /* don't change the order of handling open fds and blocked locks, since
+         * the act of opening files also reacquires granted locks in new graph.
+         */
+        fuse_handle_opened_fds (args->this, args->old_subvol, args->new_subvol);
+
+        fuse_handle_blocked_locks (args->this, args->old_subvol,
+                                   args->new_subvol);
+
+out:
+        return 0;
+}
+
+
+fuse_graph_switch_args_t *
+fuse_graph_switch_args_alloc (void)
+{
+        fuse_graph_switch_args_t *args = NULL;
+
+        args = GF_CALLOC (1, sizeof (*args), gf_fuse_mt_graph_switch_args_t);
+        if (args == NULL) {
+                goto out;
+        }
+
+out:
+        return args;
+}
+
+
+void
+fuse_graph_switch_args_destroy (fuse_graph_switch_args_t *args)
+{
+        if (args == NULL) {
+                goto out;
+        }
+
+        GF_FREE (args);
+out:
+        return;
+}
+
+
+int
+fuse_handle_graph_switch (xlator_t *this, xlator_t *old_subvol,
+                          xlator_t *new_subvol)
+{
+        call_frame_t             *frame = NULL;
+        int32_t                   ret   = -1;
+        fuse_graph_switch_args_t *args  = NULL;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (frame == NULL) {
+                goto out;
+        }
+
+        args = fuse_graph_switch_args_alloc ();
+        if (args == NULL) {
+                goto out;
+        }
+
+        args->this = this;
+        args->old_subvol = old_subvol;
+        args->new_subvol = new_subvol;
+
+        ret = synctask_new (this->ctx->env, fuse_graph_switch_task, NULL, frame,
+                            args);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "starting sync-task to "
+                        "handle graph switch failed");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (args != NULL) {
+                fuse_graph_switch_args_destroy (args);
+        }
+
+        if (frame != NULL) {
+                STACK_DESTROY (frame->root);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_graph_sync (xlator_t *this)
+{
+        fuse_private_t *priv                = NULL;
+        int             need_first_lookup   = 0;
+        int             ret                 = 0;
+        xlator_t       *old_subvol          = NULL, *new_subvol = NULL;
+        uint64_t        winds_on_old_subvol = 0;
+
+        priv = this->private;
+
+        pthread_mutex_lock (&priv->sync_mutex);
+        {
+                if (!priv->next_graph)
+                        goto unlock;
+
+                old_subvol = priv->active_subvol;
+                new_subvol = priv->active_subvol = priv->next_graph->top;
+                priv->next_graph = NULL;
+                need_first_lookup = 1;
+
+                while (!priv->event_recvd) {
+                        ret = pthread_cond_wait (&priv->sync_cond,
+                                                 &priv->sync_mutex);
+                        if (ret != 0) {
+                                  gf_log (this->name, GF_LOG_DEBUG,
+                                          "timedwait returned non zero value "
+                                          "ret: %d errno: %d", ret, errno);
+                                  break;
+                        }
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&priv->sync_mutex);
+
+        if (need_first_lookup) {
+                fuse_first_lookup (this);
+        }
+
+        if ((old_subvol != NULL) && (new_subvol != NULL)) {
+                fuse_handle_graph_switch (this, old_subvol, new_subvol);
+
+                pthread_mutex_lock (&priv->sync_mutex);
+                {
+                        old_subvol->switched = 1;
+                        winds_on_old_subvol = old_subvol->winds;
+                }
+                pthread_mutex_unlock (&priv->sync_mutex);
+
+                if (winds_on_old_subvol == 0) {
+                        xlator_notify (old_subvol, GF_EVENT_PARENT_DOWN,
+                                       old_subvol, NULL);
+                }
+        }
+
+        return 0;
+}
+
+int
+fuse_get_mount_status (xlator_t *this)
+{
+        int             kid_status = -1;
+        fuse_private_t *priv = this->private;
+
+        if (read(priv->status_pipe[0],&kid_status, sizeof(kid_status)) < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "could not get mount status");
+                kid_status = -1;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "mount status is %d", kid_status);
+
+        close(priv->status_pipe[0]);
+        close(priv->status_pipe[1]);
+        return kid_status;
+}
 
 static void *
 fuse_thread_proc (void *data)
 {
-	char *mount_point = NULL;
-        xlator_t *this = data;
-        fuse_private_t *priv = this->private;
-        int32_t res = 0;
-        data_t *buf = priv->buf;
-        int32_t ref = 0;
-        size_t chan_size = fuse_chan_bufsize (priv->ch);
-        char *recvbuf = CALLOC (1, chan_size);
-        ERR_ABORT (recvbuf);
+        char                     *mount_point = NULL;
+        xlator_t                 *this = NULL;
+        fuse_private_t           *priv = NULL;
+        ssize_t                   res = 0;
+        struct iobuf             *iobuf = NULL;
+        fuse_in_header_t         *finh;
+        struct iovec              iov_in[2];
+        void                     *msg = NULL;
+        const size_t              msg0_size = sizeof (*finh) + 128;
+        fuse_handler_t          **fuse_ops = NULL;
+        struct pollfd             pfd[2] = {{0,}};
+        gf_boolean_t              mount_finished = _gf_false;
+
+        this = data;
+        priv = this->private;
+        fuse_ops = priv->fuse_ops;
+
+        THIS = this;
+
+        iov_in[0].iov_len = sizeof (*finh) + sizeof (struct fuse_write_in);
+        iov_in[1].iov_len = ((struct iobuf_pool *)this->ctx->iobuf_pool)
+                              ->default_page_size;
+        priv->msg0_len_p = &iov_in[0].iov_len;
+
+        for (;;) {
+                /* THIS has to be reset here */
+                THIS = this;
+
+                if (!mount_finished) {
+                        memset(pfd,0,sizeof(pfd));
+                        pfd[0].fd = priv->status_pipe[0];
+                        pfd[0].events = POLLIN | POLLHUP | POLLERR;
+                        pfd[1].fd = priv->fd;
+                        pfd[1].events = POLLIN | POLLHUP | POLLERR;
+                        if (poll(pfd,2,-1) < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "poll error %s", strerror(errno));
+                                break;
+                        }
+                        if (pfd[0].revents & POLLIN) {
+                                if (fuse_get_mount_status(this) != 0) {
+                                        break;
+                                }
+                                mount_finished = _gf_true;
+                        }
+                        else if (pfd[0].revents) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "mount pipe closed without status");
+                                break;
+                        }
+                        if (!pfd[1].revents) {
+                                continue;
+                        }
+                }
 
-        while (!fuse_session_exited (priv->se)) {
+                /*
+                 * We don't want to block on readv while we're still waiting
+                 * for mount status.  That means we only want to get here if
+                 * mount_status is true (meaning that our wait completed
+                 * already) or if we already called poll(2) on priv->fd to
+                 * make sure it's ready.
+                 */
+
+                if (priv->init_recvd)
+                        fuse_graph_sync (this);
+
+                /* TODO: This place should always get maximum supported buffer
+                   size from 'fuse', which is as of today 128KB. If we bring in
+                   support for higher block sizes support, then we should be
+                   changing this one too */
+                iobuf = iobuf_get (this->ctx->iobuf_pool);
+
+                /* Add extra 128 byte to the first iov so that it can
+                 * accommodate "ordinary" non-write requests. It's not
+                 * guaranteed to be big enough, as SETXATTR and namespace
+                 * operations with very long names may grow behind it,
+                 * but it's good enough in most cases (and we can handle
+                 * rest via realloc).
+                 */
+                iov_in[0].iov_base = GF_CALLOC (1, msg0_size,
+                                                gf_fuse_mt_iov_base);
+
+                if (!iobuf || !iov_in[0].iov_base) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Out of memory");
+                        if (iobuf)
+                                iobuf_unref (iobuf);
+                        GF_FREE (iov_in[0].iov_base);
+                        sleep (10);
+                        continue;
+                }
 
+                iov_in[1].iov_base = iobuf->ptr;
 
-                res = fuse_chan_receive (priv->ch,
-                                         recvbuf,
-                                         chan_size);
+                res = readv (priv->fd, iov_in, 2);
 
                 if (res == -1) {
+                        if (errno == ENODEV || errno == EBADF) {
+                                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+                                        "terminating upon getting %s when "
+                                        "reading /dev/fuse",
+                                        errno == ENODEV ? "ENODEV" : "EBADF");
+                                fuse_log_eh (this, "glusterfs-fuse: terminating"
+                                             " upon getting %s when "
+                                             "reading /dev/fuse",
+                                             errno == ENODEV ? "ENODEV":
+                                             "EBADF");
+                                break;
+                        }
                         if (errno != EINTR) {
-                                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                        "fuse_chan_receive() returned -1 (%d)", errno);
+                                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                        "read from /dev/fuse returned -1 (%s)",
+                                        strerror (errno));
+                                fuse_log_eh (this, "glusterfs-fuse: read from "
+                                             "/dev/fuse returned -1 (%s)",
+                                             strerror (errno));
                         }
-                        if (errno == ENODEV)
-                                break;
-                        continue;
+
+                        goto cont_err;
+                }
+                if (res < sizeof (finh)) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "short read on /dev/fuse");
+                        fuse_log_eh (this, "glusterfs-fuse: short read on "
+                                     "/dev/fuse");
+                        break;
+                }
+
+                finh = (fuse_in_header_t *)iov_in[0].iov_base;
+
+                if (res != finh->len
+#ifdef GF_DARWIN_HOST_OS
+                    /* work around fuse4bsd/MacFUSE msg size miscalculation bug,
+                     * that is, payload size is not taken into account for
+                     * buffered writes
+                     */
+                    && !(finh->opcode == FUSE_WRITE &&
+                         finh->len == sizeof(*finh) + sizeof(struct fuse_write_in) &&
+                         res == finh->len + ((struct fuse_write_in *)(finh + 1))->size)
+#endif
+                   ) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "inconsistent read on /dev/fuse");
+                        fuse_log_eh (this, "glusterfs-fuse: inconsistent read "
+                                     "on /dev/fuse");
+                        break;
                 }
 
-                buf = priv->buf;
+                priv->iobuf = iobuf;
+
+                if (finh->opcode == FUSE_WRITE)
+                        msg = iov_in[1].iov_base;
+                else {
+                        if (res > msg0_size) {
+                                void *b = GF_REALLOC (iov_in[0].iov_base, res);
+                                if (b) {
+                                        iov_in[0].iov_base = b;
+                                        finh = (fuse_in_header_t *)
+                                                 iov_in[0].iov_base;
+                                }
+                                else {
+                                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                                "Out of memory");
+                                        send_fuse_err (this, finh, ENOMEM);
 
-                if (res && res != -1) {
-                        if (buf->len < (res)) {
-                                if (buf->data) {
-                                        FREE (buf->data);
-                                        buf->data = NULL;
+                                        goto cont_err;
                                 }
-                                buf->data = CALLOC (1, res);
-                                ERR_ABORT (buf->data);
-                                buf->len = res;
                         }
-                        memcpy (buf->data, recvbuf, res); // evil evil
 
-                        fuse_session_process (priv->se,
-                                              buf->data,
-                                              res,
-                                              priv->ch);
+                        if (res > iov_in[0].iov_len)
+                                memcpy (iov_in[0].iov_base + iov_in[0].iov_len,
+                                        iov_in[1].iov_base,
+                                        res - iov_in[0].iov_len);
+
+                        msg = finh + 1;
                 }
+                if (priv->uid_map_root &&
+                    finh->uid == priv->uid_map_root)
+                        finh->uid = 0;
 
-                LOCK (&buf->lock);
-                ref = buf->refcount;
-                UNLOCK (&buf->lock);
-                if (1) {
-                        data_unref (buf);
+                if (finh->opcode >= FUSE_OP_HIGH)
+                        /* turn down MacFUSE specific messages */
+                        fuse_enosys (this, finh, msg);
+                else
+                        fuse_ops[finh->opcode] (this, finh, msg);
 
-                        priv->buf = data_ref (data_from_dynptr (NULL, 0));
-                }
+                iobuf_unref (iobuf);
+                continue;
+
+ cont_err:
+                iobuf_unref (iobuf);
+                GF_FREE (iov_in[0].iov_base);
         }
-	if (dict_get (this->options, ZR_MOUNTPOINT_OPT))
-		mount_point = data_to_str (dict_get (this->options, 
-						     ZR_MOUNTPOINT_OPT));
-	if (mount_point) {
-		gf_log (this->name, GF_LOG_WARNING, 
-			"unmounting %s", mount_point);
-		dict_del (this->options, ZR_MOUNTPOINT_OPT);
-	}
-	fuse_session_remove_chan (priv->ch);
-	fuse_session_destroy (priv->se);
-	//  fuse_unmount (priv->mount_point, priv->ch);
-	
-	raise (SIGTERM);
-	
+
+        /*
+         * We could be in all sorts of states with respect to iobuf and iov_in
+         * by the time we get here, and it's just not worth untangling them if
+         * we're about to kill ourselves anyway.
+         */
+
+        if (dict_get (this->options, ZR_MOUNTPOINT_OPT))
+                mount_point = data_to_str (dict_get (this->options,
+                                                     ZR_MOUNTPOINT_OPT));
+        if (mount_point) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "unmounting %s", mount_point);
+        }
+
+        /* Kill the whole process, not just this thread. */
+        kill (getpid(), SIGTERM);
         return NULL;
 }
 
 
 int32_t
-notify (xlator_t *this, int32_t event,
-        void *data, ...)
+fuse_itable_dump (xlator_t  *this)
 {
-  
-        switch (event)
+        if (!this)
+                 return -1;
+
+        gf_proc_dump_add_section("xlator.mount.fuse.itable");
+        inode_table_dump(this->itable, "xlator.mount.fuse.itable");
+
+        return 0;
+}
+
+int32_t
+fuse_priv_dump (xlator_t  *this)
+{
+        fuse_private_t  *private = NULL;
+
+        if (!this)
+                return -1;
+
+        private = this->private;
+
+        if (!private)
+                return -1;
+
+        gf_proc_dump_add_section("xlator.mount.fuse.priv");
+
+        gf_proc_dump_write("fd", "%d", private->fd);
+        gf_proc_dump_write("proto_minor", "%u",
+                            private->proto_minor);
+        gf_proc_dump_write("volfile", "%s",
+                            private->volfile?private->volfile:"None");
+        gf_proc_dump_write("volfile_size", "%d",
+                            private->volfile_size);
+        gf_proc_dump_write("mount_point", "%s",
+                            private->mount_point);
+        gf_proc_dump_write("iobuf", "%u",
+                            private->iobuf);
+        gf_proc_dump_write("fuse_thread_started", "%d",
+                            (int)private->fuse_thread_started);
+        gf_proc_dump_write("direct_io_mode", "%d",
+                            private->direct_io_mode);
+        gf_proc_dump_write("entry_timeout", "%lf",
+                            private->entry_timeout);
+        gf_proc_dump_write("attribute_timeout", "%lf",
+                            private->attribute_timeout);
+        gf_proc_dump_write("init_recvd", "%d",
+                            (int)private->init_recvd);
+        gf_proc_dump_write("strict_volfile_check", "%d",
+                            (int)private->strict_volfile_check);
+        gf_proc_dump_write("reverse_thread_started", "%d",
+                           (int)private->reverse_fuse_thread_started);
+        gf_proc_dump_write("use_readdirp", "%d", private->use_readdirp);
+
+        return 0;
+}
+
+int
+fuse_history_dump (xlator_t *this)
+{
+        int      ret    = -1;
+        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
+
+        GF_VALIDATE_OR_GOTO ("fuse", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
+
+        gf_proc_dump_build_key (key_prefix, "xlator.mount.fuse",
+                                "history");
+        gf_proc_dump_add_section (key_prefix);
+        eh_dump (this->history, NULL, dump_history_fuse);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+dump_history_fuse (circular_buffer_t *cb, void *data)
+{
+        char       *string   =  NULL;
+        struct tm  *tm       =  NULL;
+        char       timestr[256] = {0,};
+
+        string = (char *)cb->data;
+        tm = localtime (&cb->tv.tv_sec);
+
+        if (tm) {
+                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
+                gf_proc_dump_write ("TIME", "%s", timestr);
+        }
+
+        gf_proc_dump_write ("message", "%s\n", string);
+
+        return 0;
+}
+
+int
+fuse_graph_setup (xlator_t *this, glusterfs_graph_t *graph)
+{
+        inode_table_t  *itable = NULL;
+        int             ret = 0;
+        fuse_private_t *priv = NULL;
+
+        priv = this->private;
+
+        /* handle the case of more than one CHILD_UP on same graph */
+        if (priv->active_subvol == graph->top)
+                return 0; /* This is a valid case */
+
+        if (graph->used)
+                return 0;
+
+        graph->used = 1;
+
+        itable = inode_table_new (0, graph->top);
+        if (!itable)
+                return -1;
+
+        ((xlator_t *)graph->top)->itable = itable;
+
+        pthread_mutex_lock (&priv->sync_mutex);
         {
-        case GF_EVENT_CHILD_UP:
+                priv->next_graph = graph;
+                priv->event_recvd = 0;
 
-#ifndef GF_DARWIN_HOST_OS
-		/* 
-		 * This is because macfuse sends statfs() once the fuse thread
-                 * gets activated, and by that time if the client is not 
-		 * connected, it give 'Device not configured' error. Hence, 
-		 * create thread only when client sends CHILD_UP (ie, client 
-		 * is connected).
-		 */
+                pthread_cond_signal (&priv->sync_cond);
+        }
+        pthread_mutex_unlock (&priv->sync_mutex);
 
-                /* TODO: somehow, try to get the mountpoint active as soon as 
-		 * init() is complete, so that the hang effect when the 
-		 * server is not not started is removed. 
-		 */
-		
-		/* This code causes problem with 'automount' too */
-		/* case GF_EVENT_CHILD_CONNECTING: */
-#endif /* DARWIN */
+        gf_log ("fuse", GF_LOG_INFO, "switched to graph %d",
+                ((graph) ? graph->id : 0));
+
+        return ret;
+}
+
+
+int
+notify (xlator_t *this, int32_t event, void *data, ...)
+{
+        int32_t             ret     = 0;
+        fuse_private_t     *private = NULL;
+        glusterfs_graph_t  *graph = NULL;
+
+        private = this->private;
+
+        graph = data;
+
+        gf_log ("fuse", GF_LOG_DEBUG, "got event %d on graph %d",
+                event, ((graph) ? graph->id : 0));
+
+        switch (event)
+        {
+        case GF_EVENT_GRAPH_NEW:
+                break;
 
+        case GF_EVENT_CHILD_UP:
+        case GF_EVENT_CHILD_DOWN:
+        case GF_EVENT_CHILD_CONNECTING:
         {
-                fuse_private_t *private = this->private;
-                int32_t ret = 0;
+                if (graph) {
+                        ret = fuse_graph_setup (this, graph);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to setup the graph");
+                }
 
-                if (!private->fuse_thread_started)
-                {
-                        private->fuse_thread_started = 1;
+                if ((event == GF_EVENT_CHILD_UP)
+                    || (event == GF_EVENT_CHILD_DOWN)) {
+                        pthread_mutex_lock (&private->sync_mutex);
+                        {
+                                private->event_recvd = 1;
+                                pthread_cond_broadcast (&private->sync_cond);
+                        }
+                        pthread_mutex_unlock (&private->sync_mutex);
+                }
 
-                        ret = pthread_create (&private->fuse_thread, NULL,
-                                              fuse_thread_proc, this);
+                if (!private->fuse_thread_started) {
+                        private->fuse_thread_started = 1;
 
-                        if (ret != 0)
-                                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                        "pthread_create() failed (%s)", strerror (errno));
-                        assert (ret == 0);
+                        ret = gf_thread_create (&private->fuse_thread, NULL,
+						fuse_thread_proc, this);
+                        if (ret != 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "pthread_create() failed (%s)",
+                                        strerror (errno));
+                                break;
+                        }
                 }
+
                 break;
         }
-        case GF_EVENT_PARENT_UP:
+
+        case GF_EVENT_AUTH_FAILED:
         {
-                default_notify (this, GF_EVENT_PARENT_UP, data);
+                /* Authentication failure is an error and glusterfs should stop */
+               gf_log (this->name, GF_LOG_ERROR, "Server authenication failed. Shutting down.");
+               fini (this);
+               break;
         }
+
         default:
                 break;
         }
-        return 0;
+
+        return ret;
 }
 
-int 
-init (xlator_t *this_xl)
+int32_t
+mem_acct_init (xlator_t *this)
 {
-	int ret = 0;
-	dict_t *options = NULL;
-	char *value_string = NULL;
-	fuse_private_t *priv = NULL;
-	struct stat stbuf = {0,};
+        int     ret = -1;
 
-#ifdef GF_DARWIN_HOST_OS
-        int fuse_argc = 9;
-	char *fuse_argv[] = {"glusterfs",
-			     "-o", "allow_other",
-			     "-o", "default_permissions",
-			     "-o", "fsname=glusterfs",
-			     "-o", "local",
-			     NULL};
-
-#elif GF_LINUX_HOST_OS /* ! DARWIN_OS */
-        int fuse_argc = 19;
-	
-	char *fuse_argv[] = {"glusterfs",
-			     "-o", "nonempty",
-			     "-o", "max_readahead=1048576",
-			     "-o", "max_read=1048576",
-			     "-o", "max_write=1048576",
-			     "-o", "allow_other",
-			     "-o", "default_permissions",
-			     "-o", "fsname=glusterfs",
-			     "-o", "dev",
-			     "-o", "suid",
-			     NULL};
-
-#else /* BSD || SOLARIS */
-	/* BSD fuse doesn't support '-o dev', '-o nonempty' option */
-        int fuse_argc = 15;
-	
-	char *fuse_argv[] = {"glusterfs",
-			     "-o", "max_readahead=1048576",
-			     "-o", "max_read=1048576",
-			     "-o", "max_write=1048576",
-			     "-o", "allow_other",
-			     "-o", "default_permissions",
-			     "-o", "fsname=glusterfs",
-			     "-o", "suid",
-			     NULL};
-
-#endif /* ! DARWIN_OS || ! LINUX */
-        struct fuse_args args = FUSE_ARGS_INIT (fuse_argc, fuse_argv);
-	
-	if (this_xl == NULL)
-		return -1;
-	
-	if (this_xl->options == NULL)
-		return -1;
-	
-	options = this_xl->options;
-	
-	if (this_xl->name == NULL)
-		this_xl->name = strdup ("fuse");
-	
-        priv = CALLOC (1, sizeof (*priv));
-        ERR_ABORT (priv);
-        this_xl->private = (void *) priv;
+        if (!this)
+                return ret;
 
+        ret = xlator_mem_acct_init (this, gf_fuse_mt_end + 1);
 
-#ifdef GF_DARWIN_HOST_OS
-	if (dict_get (options, "macfuse-local")) {
-		/* This way, GlusterFS will be detected as 'servers' instead
-		 *  of 'devices'. This method is useful if you want to do 
-		 * 'umount <mount_point>' over network,  instead of 'eject'ing
-		 * it from desktop. Works better for servers 
-		 */
-		/* Make the '-o local' in argv as NULL, so that its not 
-		   in effect */
-		fuse_argv[--args.argc] = NULL;
-		fuse_argv[--args.argc] = NULL;
-	}
-#endif /* ! DARWIN */
-
-	/* get options from option dictionary */
-	ret = dict_get_str (options, ZR_MOUNTPOINT_OPT, &value_string);
-	if (value_string == NULL) {
-                gf_log ("fuse", GF_LOG_ERROR, 
-			"mandatory option mountpoint is not specified");
-		return -1;
-	}
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                                "failed");
+                return ret;
+        }
 
-	if (stat (value_string, &stbuf) != 0) {
-		if (errno == ENOENT) {
-			gf_log (this_xl->name, GF_LOG_ERROR ,
-				"%s %s does not exist",
-				ZR_MOUNTPOINT_OPT, value_string);
-		} else if (errno == ENOTCONN) {
-			gf_log (this_xl->name, GF_LOG_ERROR ,
-				"mountpoint %s seems to have a stale "
-				"mount, run 'umount %s' and try again",
-				value_string, value_string);
-		} else {
-			gf_log (this_xl->name, GF_LOG_ERROR ,
-				"%s %s : stat returned %s",
-				ZR_MOUNTPOINT_OPT,
-				value_string, strerror (errno));
-		}
-		return -1;
-	}
-	
-	if (S_ISDIR (stbuf.st_mode) == 0) {
-		gf_log (this_xl->name, GF_LOG_ERROR ,
-			"%s %s is not a directory",
-			ZR_MOUNTPOINT_OPT, value_string);
-		return -1;
-	}
-	priv->mount_point = strdup (value_string);
-	
-	
-	ret = dict_get_double (options, "attribute-timeout", 
-			       &priv->attribute_timeout);
-	if (!priv->attribute_timeout)
-		priv->attribute_timeout = 1.0; /* default */
-	
-	ret = dict_get_double (options, "entry-timeout", 
-			       &priv->entry_timeout);
-	if (!priv->entry_timeout)
-		priv->entry_timeout = 1.0; /* default */
-	
-
-	priv->direct_io_mode = 1;
-	ret = dict_get_str (options, ZR_DIRECT_IO_OPT, &value_string);
-	if (value_string) {
-		ret = gf_string2boolean (value_string, &priv->direct_io_mode);
-	}
-	
-        priv->ch = fuse_mount (priv->mount_point, &args);
-        if (priv->ch == NULL) {
-                if (errno == ENOTCONN) {
+        return ret;
+}
+
+
+static fuse_handler_t *fuse_std_ops[FUSE_OP_HIGH] = {
+        [FUSE_LOOKUP]      = fuse_lookup,
+        [FUSE_FORGET]      = fuse_forget,
+        [FUSE_GETATTR]     = fuse_getattr,
+        [FUSE_SETATTR]     = fuse_setattr,
+        [FUSE_READLINK]    = fuse_readlink,
+        [FUSE_SYMLINK]     = fuse_symlink,
+        [FUSE_MKNOD]       = fuse_mknod,
+        [FUSE_MKDIR]       = fuse_mkdir,
+        [FUSE_UNLINK]      = fuse_unlink,
+        [FUSE_RMDIR]       = fuse_rmdir,
+        [FUSE_RENAME]      = fuse_rename,
+        [FUSE_LINK]        = fuse_link,
+        [FUSE_OPEN]        = fuse_open,
+        [FUSE_READ]        = fuse_readv,
+        [FUSE_WRITE]       = fuse_write,
+        [FUSE_STATFS]      = fuse_statfs,
+        [FUSE_RELEASE]     = fuse_release,
+        [FUSE_FSYNC]       = fuse_fsync,
+        [FUSE_SETXATTR]    = fuse_setxattr,
+        [FUSE_GETXATTR]    = fuse_getxattr,
+        [FUSE_LISTXATTR]   = fuse_listxattr,
+        [FUSE_REMOVEXATTR] = fuse_removexattr,
+        [FUSE_FLUSH]       = fuse_flush,
+        [FUSE_INIT]        = fuse_init,
+        [FUSE_OPENDIR]     = fuse_opendir,
+        [FUSE_READDIR]     = fuse_readdir,
+        [FUSE_RELEASEDIR]  = fuse_releasedir,
+        [FUSE_FSYNCDIR]    = fuse_fsyncdir,
+        [FUSE_GETLK]       = fuse_getlk,
+        [FUSE_SETLK]       = fuse_setlk,
+        [FUSE_SETLKW]      = fuse_setlk,
+        [FUSE_ACCESS]      = fuse_access,
+        [FUSE_CREATE]      = fuse_create,
+     /* [FUSE_INTERRUPT] */
+     /* [FUSE_BMAP] */
+        [FUSE_DESTROY]     = fuse_destroy,
+     /* [FUSE_IOCTL] */
+     /* [FUSE_POLL] */
+     /* [FUSE_NOTIFY_REPLY] */
+	[FUSE_BATCH_FORGET]= fuse_batch_forget,
+	[FUSE_FALLOCATE]   = fuse_fallocate,
+	[FUSE_READDIRPLUS] = fuse_readdirp,
+};
+
+
+static fuse_handler_t *fuse_dump_ops[FUSE_OP_HIGH];
+
+
+static void
+fuse_dumper (xlator_t *this, fuse_in_header_t *finh, void *msg)
+{
+        fuse_private_t *priv = NULL;
+        struct iovec diov[3];
+        char r = 'R';
+        int ret = 0;
+
+        priv = this->private;
+
+        diov[0].iov_base = &r;
+        diov[0].iov_len  = 1;
+        diov[1].iov_base = finh;
+        diov[1].iov_len  = sizeof (*finh);
+        diov[2].iov_base = msg;
+        diov[2].iov_len  = finh->len - sizeof (*finh);
+
+        pthread_mutex_lock (&priv->fuse_dump_mutex);
+        ret = writev (priv->fuse_dump_fd, diov, 3);
+        pthread_mutex_unlock (&priv->fuse_dump_mutex);
+        if (ret == -1)
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "failed to dump fuse message (R): %s",
+                        strerror (errno));
+
+        priv->fuse_ops0[finh->opcode] (this, finh, msg);
+}
+
+
+int
+init (xlator_t *this_xl)
+{
+        int                ret = 0;
+        dict_t            *options = NULL;
+        char              *value_string = NULL;
+        cmd_args_t        *cmd_args = NULL;
+        char              *fsname = NULL;
+        fuse_private_t    *priv = NULL;
+        struct stat        stbuf = {0,};
+        int                i = 0;
+        int                xl_name_allocated = 0;
+        int                fsname_allocated = 0;
+        glusterfs_ctx_t   *ctx = NULL;
+        gf_boolean_t       sync_to_mount = _gf_false;
+        gf_boolean_t       fopen_keep_cache = _gf_false;
+        unsigned long      mntflags = 0;
+        char              *mnt_args = NULL;
+        eh_t              *event = NULL;
+
+        if (this_xl == NULL)
+                return -1;
+
+        if (this_xl->options == NULL)
+                return -1;
+
+        ctx = this_xl->ctx;
+        if (!ctx)
+                return -1;
+
+        options = this_xl->options;
+
+        if (this_xl->name == NULL) {
+                this_xl->name = gf_strdup ("fuse");
+                if (!this_xl->name) {
                         gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "A stale mount present on %s. "
-				"run 'umount %s' and try again",
-                                priv->mount_point, 
-				priv->mount_point);
+                                "Out of memory");
+
+                        goto cleanup_exit;
+                }
+                xl_name_allocated = 1;
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_fuse_mt_fuse_private_t);
+        if (!priv) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                        "Out of memory");
+
+                goto cleanup_exit;
+        }
+        this_xl->private = (void *) priv;
+        priv->mount_point = NULL;
+        priv->fd = -1;
+        priv->revchan_in = -1;
+        priv->revchan_out = -1;
+
+        /* get options from option dictionary */
+        ret = dict_get_str (options, ZR_MOUNTPOINT_OPT, &value_string);
+        if (ret == -1 || value_string == NULL) {
+                gf_log ("fuse", GF_LOG_ERROR,
+                        "Mandatory option 'mountpoint' is not specified.");
+                goto cleanup_exit;
+        }
+
+        if (stat (value_string, &stbuf) != 0) {
+                if (errno == ENOENT) {
+                        gf_log (this_xl->name, GF_LOG_ERROR,
+                                "%s %s does not exist",
+                                ZR_MOUNTPOINT_OPT, value_string);
+                } else if (errno == ENOTCONN) {
+                        gf_log (this_xl->name, GF_LOG_ERROR,
+                                "Mountpoint %s seems to have a stale "
+                                "mount, run 'umount %s' and try again.",
+                                value_string, value_string);
                 } else {
-                        if (errno == ENOENT) {
-                                gf_log ("glusterfs-fuse", GF_LOG_ERROR, 
-                                        "unable to mount on %s. run "
-					"'modprobe fuse' and try again", 
-                                        priv->mount_point);
-                        } else {
-                                gf_log ("glusterfs-fuse", GF_LOG_ERROR, 
-                                        "fuse_mount() failed with error %s "
-					"on mount point %s", 
-                                        strerror (errno), 
-					priv->mount_point);
-                        }
+                        gf_log (this_xl->name, GF_LOG_DEBUG,
+                                "%s %s : stat returned %s",
+                                ZR_MOUNTPOINT_OPT,
+                                value_string, strerror (errno));
                 }
-                
                 goto cleanup_exit;
         }
-        
-        priv->se = fuse_lowlevel_new (&args, &fuse_ops, 
-				      sizeof (fuse_ops), this_xl);
-        if (priv->se == NULL) {
+
+        if (S_ISDIR (stbuf.st_mode) == 0) {
+                gf_log (this_xl->name, GF_LOG_ERROR,
+                        "%s %s is not a directory",
+                        ZR_MOUNTPOINT_OPT, value_string);
+                goto cleanup_exit;
+        }
+        priv->mount_point = gf_strdup (value_string);
+        if (!priv->mount_point) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "fuse_lowlevel_new() failed with error %s on "
-			"mount point %s", 
-                        strerror (errno), priv->mount_point);
-                goto umount_exit;
+                        "Out of memory");
+
+                goto cleanup_exit;
         }
-        
-        ret = fuse_set_signal_handlers (priv->se);
-        if (ret == -1) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR, 
-                        "fuse_set_signal_handlers() failed on mount point %s", 
-                        priv->mount_point);
-                goto umount_exit;
-        }
-        
-        fuse_opt_free_args (&args);
-        
-        fuse_session_add_chan (priv->se, priv->ch);
-        
-        priv->fd = fuse_chan_fd (priv->ch);
-        priv->buf = data_ref (data_from_dynptr (NULL, 0));
-
-        this_xl->ctx->top = this_xl;
+
+        GF_OPTION_INIT ("attribute-timeout", priv->attribute_timeout, double,
+                        cleanup_exit);
+
+        GF_OPTION_INIT ("entry-timeout", priv->entry_timeout, double,
+                        cleanup_exit);
+
+        GF_OPTION_INIT ("negative-timeout", priv->negative_timeout, double,
+                        cleanup_exit);
+
+        GF_OPTION_INIT ("client-pid", priv->client_pid, int32, cleanup_exit);
+        /* have to check & register the presence of client-pid manually */
+        priv->client_pid_set = !!dict_get (this_xl->options, "client-pid");
+
+        GF_OPTION_INIT ("uid-map-root", priv->uid_map_root, uint32,
+                        cleanup_exit);
+
+        priv->direct_io_mode = 2;
+        ret = dict_get_str (options, ZR_DIRECT_IO_OPT, &value_string);
+        if (ret == 0) {
+                ret = gf_string2boolean (value_string, &priv->direct_io_mode);
+                GF_ASSERT (ret == 0);
+        }
+
+        GF_OPTION_INIT (ZR_STRICT_VOLFILE_CHECK, priv->strict_volfile_check,
+                        bool, cleanup_exit);
+
+        GF_OPTION_INIT ("acl", priv->acl, bool, cleanup_exit);
+
+        if (priv->uid_map_root)
+                priv->acl = 1;
+
+        GF_OPTION_INIT ("selinux", priv->selinux, bool, cleanup_exit);
+
+        GF_OPTION_INIT ("read-only", priv->read_only, bool, cleanup_exit);
+
+        GF_OPTION_INIT ("enable-ino32", priv->enable_ino32, bool, cleanup_exit);
+
+        GF_OPTION_INIT ("use-readdirp", priv->use_readdirp, bool, cleanup_exit);
+
+        priv->fuse_dump_fd = -1;
+        ret = dict_get_str (options, "dump-fuse", &value_string);
+        if (ret == 0) {
+                ret = unlink (value_string);
+                if (ret != -1 || errno == ENOENT)
+                        ret = open (value_string, O_RDWR|O_CREAT|O_EXCL,
+                                    S_IRUSR|S_IWUSR);
+                if (ret == -1) {
+                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                "cannot open fuse dump file %s",
+                                 value_string);
+
+                        goto cleanup_exit;
+                }
+                priv->fuse_dump_fd = ret;
+        }
+
+        sync_to_mount = _gf_false;
+        ret = dict_get_str (options, "sync-to-mount", &value_string);
+        if (ret == 0) {
+                ret = gf_string2boolean (value_string,
+                                         &sync_to_mount);
+                GF_ASSERT (ret == 0);
+        }
+
+	priv->fopen_keep_cache = 2;
+	if (dict_get (options, "fopen-keep-cache")) {
+		GF_OPTION_INIT("fopen-keep-cache", fopen_keep_cache, bool,
+			       cleanup_exit);
+		priv->fopen_keep_cache = fopen_keep_cache;
+	}
+
+        GF_OPTION_INIT("gid-timeout", priv->gid_cache_timeout, int32,
+                cleanup_exit);
+
+        GF_OPTION_INIT ("fuse-mountopts", priv->fuse_mountopts, str, cleanup_exit);
+
+        if (gid_cache_init(&priv->gid_cache, priv->gid_cache_timeout) < 0) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR, "Failed to initialize "
+                        "group cache.");
+                goto cleanup_exit;
+        }
+
+        /* default values seemed to work fine during testing */
+        GF_OPTION_INIT ("background-qlen", priv->background_qlen, int32,
+                        cleanup_exit);
+        GF_OPTION_INIT ("congestion-threshold", priv->congestion_threshold,
+                        int32, cleanup_exit);
+
+        /* user has set only background-qlen, not congestion-threshold,
+           use the fuse kernel driver formula to set congestion. ie, 75% */
+        if (dict_get (this_xl->options, "background-qlen") &&
+            !dict_get (this_xl->options, "congestion-threshold")) {
+                priv->congestion_threshold = (priv->background_qlen * 3) / 4;
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control as 75%% of "
+                        "background-queue length (ie, (.75 * %d) = %d",
+                        priv->background_qlen, priv->congestion_threshold);
+        }
+
+        /* congestion should not be higher than background queue length */
+        if (priv->congestion_threshold > priv->background_qlen) {
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control same as "
+                        "background-queue length (%d)",
+                        priv->background_qlen);
+                priv->congestion_threshold = priv->background_qlen;
+        }
+
+        cmd_args = &this_xl->ctx->cmd_args;
+        fsname = cmd_args->volfile;
+        if (!fsname && cmd_args->volfile_server) {
+                if (cmd_args->volfile_id) {
+                        fsname = GF_MALLOC (
+                                   strlen (cmd_args->volfile_server) + 1 +
+                                   strlen (cmd_args->volfile_id) + 1,
+                                   gf_fuse_mt_fuse_private_t);
+                        if (!fsname) {
+                                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
+                                        "Out of memory");
+                                goto cleanup_exit;
+                        }
+                        fsname_allocated = 1;
+                        strcpy (fsname, cmd_args->volfile_server);
+                        strcat (fsname, ":");
+                        strcat (fsname, cmd_args->volfile_id);
+                } else
+                        fsname = cmd_args->volfile_server;
+        }
+        if (!fsname)
+                fsname = "glusterfs";
+
+        priv->fdtable = gf_fd_fdtable_alloc ();
+        if (priv->fdtable == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
+                goto cleanup_exit;
+        }
+
+        if (priv->read_only)
+                mntflags |= MS_RDONLY;
+        gf_asprintf (&mnt_args, "%s%s%sallow_other,max_read=131072",
+                     priv->acl ? "" : "default_permissions,",
+                     priv->fuse_mountopts ? priv->fuse_mountopts : "",
+                     priv->fuse_mountopts ? "," : "");
+        if (!mnt_args)
+                goto cleanup_exit;
+
+        if (pipe(priv->status_pipe) < 0) {
+                gf_log (this_xl->name, GF_LOG_ERROR,
+                        "could not create pipe to separate mount process");
+                goto cleanup_exit;
+        }
+
+        priv->fd = gf_fuse_mount (priv->mount_point, fsname, mntflags, mnt_args,
+                                  sync_to_mount ? &ctx->mnt_pid : NULL,
+                                  priv->status_pipe[1]);
+        if (priv->fd == -1)
+                goto cleanup_exit;
+
+        event = eh_new (FUSE_EVENT_HISTORY_SIZE, _gf_false, NULL);
+        if (!event) {
+                gf_log (this_xl->name, GF_LOG_ERROR,
+                        "could not create a new event history");
+                goto cleanup_exit;
+        }
+
+        this_xl->history = event;
+
+        pthread_mutex_init (&priv->fuse_dump_mutex, NULL);
+        pthread_cond_init (&priv->sync_cond, NULL);
+        pthread_mutex_init (&priv->sync_mutex, NULL);
+        priv->event_recvd = 0;
+
+        for (i = 0; i < FUSE_OP_HIGH; i++) {
+                if (!fuse_std_ops[i])
+                        fuse_std_ops[i] = fuse_enosys;
+                if (!fuse_dump_ops[i])
+                        fuse_dump_ops[i] = fuse_dumper;
+        }
+        priv->fuse_ops = fuse_std_ops;
+        if (priv->fuse_dump_fd != -1) {
+                priv->fuse_ops0 = priv->fuse_ops;
+                priv->fuse_ops  = fuse_dump_ops;
+        }
+
+        if (fsname_allocated)
+                GF_FREE (fsname);
+        GF_FREE (mnt_args);
         return 0;
-        
-umount_exit: 
-        fuse_unmount (priv->mount_point, priv->ch);
+
 cleanup_exit:
-        fuse_opt_free_args (&args);
-        FREE (priv->mount_point);
-        FREE (priv);
+        if (xl_name_allocated)
+                GF_FREE (this_xl->name);
+        if (fsname_allocated)
+                GF_FREE (fsname);
+        if (priv) {
+                GF_FREE (priv->mount_point);
+                if (priv->fd != -1)
+                        close (priv->fd);
+                if (priv->fuse_dump_fd != -1)
+                        close (priv->fuse_dump_fd);
+                GF_FREE (priv);
+        }
+        GF_FREE (mnt_args);
         return -1;
 }
 
@@ -2805,51 +5408,121 @@ void
 fini (xlator_t *this_xl)
 {
         fuse_private_t *priv = NULL;
-	char *mount_point = NULL;
-	
-	if (this_xl == NULL)
-		return;
-	
-	if ((priv = this_xl->private) == NULL)
-		return;
-	
-	if (dict_get (this_xl->options, ZR_MOUNTPOINT_OPT))
-		mount_point = data_to_str (dict_get (this_xl->options, 
-						     ZR_MOUNTPOINT_OPT));
-	if (mount_point != NULL) {
-		gf_log (this_xl->name, GF_LOG_WARNING, 
-			"unmounting '%s'", mount_point);
-		
-		dict_del (this_xl->options, ZR_MOUNTPOINT_OPT);
-		fuse_session_exit (priv->se);
-		fuse_unmount (mount_point, priv->ch);
-	}
+        char *mount_point = NULL;
+
+        if (this_xl == NULL)
+                return;
+
+        if ((priv = this_xl->private) == NULL)
+                return;
+
+        if (dict_get (this_xl->options, ZR_MOUNTPOINT_OPT))
+                mount_point = data_to_str (dict_get (this_xl->options,
+                                                     ZR_MOUNTPOINT_OPT));
+        if (mount_point != NULL) {
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "Unmounting '%s'.", mount_point);
+
+                gf_fuse_unmount (mount_point, priv->fd);
+                close (priv->fuse_dump_fd);
+                dict_del (this_xl->options, ZR_MOUNTPOINT_OPT);
+        }
+        /* Process should terminate once fuse xlator is finished.
+         * Required for AUTH_FAILED event.
+         */
+        kill (getpid (), SIGTERM);
 }
 
-struct xlator_fops fops = {
-};
+struct xlator_fops fops;
 
 struct xlator_cbks cbks = {
+        .invalidate = fuse_invalidate,
+        .forget     = fuse_forget_cbk,
 };
 
-struct xlator_mops mops = {
+
+struct xlator_dumpops dumpops = {
+        .priv  = fuse_priv_dump,
+        .inode = fuse_itable_dump,
+        .history = fuse_history_dump,
 };
 
 struct volume_options options[] = {
-	{ .key  = {"direct-io-mode"}, 
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = {"macfuse-local"}, 
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = {"mountpoint", "mount-point"}, 
-	  .type = GF_OPTION_TYPE_PATH 
-	},
-	{ .key  = {"attribute-timeout"}, 
-	  .type = GF_OPTION_TYPE_DOUBLE
-	},
-	{ .key  = {"entry-timeout"}, 
-	  .type = GF_OPTION_TYPE_DOUBLE
-	},
-	{ .key = {NULL} },
+        { .key  = {"direct-io-mode"},
+          .type = GF_OPTION_TYPE_BOOL
+        },
+        { .key  = {ZR_MOUNTPOINT_OPT, "mount-point"},
+          .type = GF_OPTION_TYPE_PATH
+        },
+        { .key  = {ZR_DUMP_FUSE, "fuse-dumpfile"},
+          .type = GF_OPTION_TYPE_PATH
+        },
+        { .key  = {ZR_ATTR_TIMEOUT_OPT},
+          .type = GF_OPTION_TYPE_DOUBLE,
+          .default_value = "1.0"
+        },
+        { .key  = {ZR_ENTRY_TIMEOUT_OPT},
+          .type = GF_OPTION_TYPE_DOUBLE,
+          .default_value = "1.0"
+        },
+        { .key  = {ZR_NEGATIVE_TIMEOUT_OPT},
+          .type = GF_OPTION_TYPE_DOUBLE,
+          .default_value = "0.0"
+        },
+        { .key  = {ZR_STRICT_VOLFILE_CHECK},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key  = {"client-pid"},
+          .type = GF_OPTION_TYPE_INT
+        },
+        { .key  = {"uid-map-root"},
+          .type = GF_OPTION_TYPE_INT
+        },
+        { .key  = {"sync-to-mount"},
+          .type = GF_OPTION_TYPE_BOOL
+        },
+        { .key = {"read-only"},
+          .type = GF_OPTION_TYPE_BOOL
+        },
+        { .key = {"fopen-keep-cache"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key = {"gid-timeout"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "2"
+        },
+        { .key = {"acl"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key = {"selinux"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key = {"enable-ino32"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false"
+        },
+        { .key  = {"background-qlen"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "64",
+          .min = 16,
+          .max = (64 * GF_UNIT_KB),
+        },
+        { .key  = {"congestion-threshold"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "48",
+          .min = 12,
+          .max = (64 * GF_UNIT_KB),
+        },
+        { .key = {"fuse-mountopts"},
+          .type = GF_OPTION_TYPE_STR
+        },
+        { .key = {"use-readdirp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "yes"
+        },
+        { .key = {NULL} },
 };
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
new file mode 100644
index 000000000..34794b6ea
--- /dev/null
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -0,0 +1,537 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GF_FUSE_BRIDGE_H_
+#define _GF_FUSE_BRIDGE_H_
+
+#include <stdint.h>
+#include <signal.h>
+#include <pthread.h>
+#include <stddef.h>
+#include <dirent.h>
+#include <sys/mount.h>
+#include <sys/time.h>
+#include <fnmatch.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif /* _CONFIG_H */
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "xlator.h"
+#include "defaults.h"
+#include "common-utils.h"
+#include "statedump.h"
+
+#ifdef GF_DARWIN_HOST_OS
+/* This is MacFUSE's marker for MacFUSE-specific code */
+#define __FreeBSD__ 10
+#include "fuse_kernel_macfuse.h"
+#else
+#include "fuse_kernel.h"
+#endif
+#include "fuse-misc.h"
+#include "fuse-mount.h"
+#include "fuse-mem-types.h"
+
+#include "list.h"
+#include "dict.h"
+#include "syncop.h"
+#include "gidcache.h"
+
+#if defined(GF_LINUX_HOST_OS) || defined(__NetBSD__)
+#define FUSE_OP_HIGH (FUSE_READDIRPLUS + 1)
+#endif
+#ifdef GF_DARWIN_HOST_OS
+#define FUSE_OP_HIGH (FUSE_DESTROY + 1)
+#endif
+#define GLUSTERFS_XATTR_LEN_MAX  65536
+
+#define MAX_FUSE_PROC_DELAY 1
+
+typedef struct fuse_in_header fuse_in_header_t;
+typedef void (fuse_handler_t) (xlator_t *this, fuse_in_header_t *finh,
+                               void *msg);
+
+struct fuse_private {
+        int                  fd;
+        uint32_t             proto_minor;
+        char                *volfile;
+        size_t               volfile_size;
+        char                *mount_point;
+        struct iobuf        *iobuf;
+
+        pthread_t            fuse_thread;
+        char                 fuse_thread_started;
+
+        uint32_t             direct_io_mode;
+        size_t              *msg0_len_p;
+
+        double               entry_timeout;
+        double               negative_timeout;
+        double               attribute_timeout;
+
+        pthread_cond_t       sync_cond;
+        pthread_mutex_t      sync_mutex;
+        char                 event_recvd;
+
+        char                 init_recvd;
+
+        gf_boolean_t         strict_volfile_check;
+
+        fuse_handler_t     **fuse_ops;
+        fuse_handler_t     **fuse_ops0;
+        pthread_mutex_t      fuse_dump_mutex;
+        int                  fuse_dump_fd;
+
+        glusterfs_graph_t   *next_graph;
+        xlator_t            *active_subvol;
+
+        pid_t                client_pid;
+        gf_boolean_t         client_pid_set;
+        unsigned             uid_map_root;
+        gf_boolean_t         acl;
+        gf_boolean_t         selinux;
+        gf_boolean_t         read_only;
+	int32_t	             fopen_keep_cache;
+	int32_t		     gid_cache_timeout;
+        gf_boolean_t         enable_ino32;
+        fdtable_t           *fdtable;
+	gid_cache_t	     gid_cache;
+        char                *fuse_mountopts;
+
+        /* For fuse-reverse-validation */
+        int                  revchan_in;
+        int                  revchan_out;
+        gf_boolean_t         reverse_fuse_thread_started;
+
+        /* For communicating with separate mount thread. */
+        int                  status_pipe[2];
+
+        /* for fuse queue length and congestion threshold */
+        int background_qlen;
+        int congestion_threshold;
+
+        /* for using fuse-kernel readdirp*/
+        gf_boolean_t use_readdirp;
+};
+typedef struct fuse_private fuse_private_t;
+
+struct fuse_graph_switch_args {
+        xlator_t        *this;
+        xlator_t        *old_subvol;
+        xlator_t        *new_subvol;
+};
+typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
+
+#define INVAL_BUF_SIZE (sizeof (struct fuse_out_header) +               \
+                        max (sizeof (struct fuse_notify_inval_inode_out), \
+                             sizeof (struct fuse_notify_inval_entry_out) + \
+                             NAME_MAX + 1))
+
+#define FUSE_EVENT_HISTORY_SIZE 1024
+
+#define _FH_TO_FD(fh) ((fd_t *)(uintptr_t)(fh))
+
+#define FH_TO_FD(fh) ((_FH_TO_FD (fh))?(fd_ref (_FH_TO_FD (fh))):((fd_t *) 0))
+
+#define FUSE_FOP(state, ret, op_num, fop, args ...)                     \
+        do {                                                            \
+                call_frame_t   *frame   = NULL;                         \
+                xlator_t       *xl      = NULL;                         \
+                int32_t         op_ret  = 0, op_errno = 0;              \
+                fuse_resolve_t *resolve = NULL;                         \
+                                                                        \
+                frame = get_call_frame_for_req (state);                 \
+                if (!frame) {                                           \
+                         /* This is not completely clean, as some       \
+                          * earlier allocations might remain unfreed    \
+                          * if we return at this point, but still       \
+                          * better than trying to go on with a NULL     \
+                          * frame ...                                   \
+                          */                                            \
+                        gf_log_callingfn ("glusterfs-fuse",             \
+                                GF_LOG_ERROR,                           \
+                                "FUSE message"                          \
+                                " unique %"PRIu64" opcode %d:"          \
+                                " frame allocation failed",             \
+                                state->finh->unique,                    \
+                                state->finh->opcode);                   \
+                        free_fuse_state (state);                        \
+                        /* ideally, need to 'return', but let the */    \
+                        /* calling function take care of it */          \
+                        break;                                          \
+                }                                                       \
+                                                                        \
+                frame->root->state = state;                             \
+                frame->root->op    = op_num;                            \
+                frame->op          = op_num;                            \
+                                                                        \
+                if ( state->resolve_now ) {                             \
+                        resolve = state->resolve_now;                   \
+                } else {                                                \
+                        resolve = &(state->resolve);                    \
+                }                                                       \
+                                                                        \
+                xl = state->active_subvol;				\
+                if (!xl) {                                              \
+                        gf_log_callingfn ("glusterfs-fuse", GF_LOG_ERROR, \
+                                          "xl is NULL");                \
+                        op_errno = ENOENT;                              \
+                        op_ret = -1;                                    \
+                } else if (resolve->op_ret < 0) {                       \
+                        op_errno = resolve->op_errno;                   \
+                        op_ret = -1;                                    \
+                        if (op_num == GF_FOP_LOOKUP) {                  \
+                                gf_log ("glusterfs-fuse",               \
+                                        (op_errno == ENOENT ? GF_LOG_TRACE \
+                                         : GF_LOG_WARNING),             \
+                                        "%"PRIu64": %s() %s => -1 (%s)", \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        resolve->resolve_loc.path,      \
+                                        strerror (op_errno));           \
+                        } else {                                        \
+                                gf_log ("glusterfs-fuse",               \
+                                        GF_LOG_WARNING,                 \
+                                        "%"PRIu64": %s() inode "        \
+                                        "migration of %s failed (%s)",  \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        resolve->resolve_loc.path,      \
+                                        strerror (op_errno));           \
+                        }                                               \
+                } else if (state->resolve2.op_ret < 0) {                \
+                        op_errno = state->resolve2.op_errno;            \
+                        op_ret = -1;                                    \
+                        gf_log ("glusterfs-fuse",                       \
+                                GF_LOG_WARNING,                         \
+                                "%"PRIu64": %s() inode "                \
+                                "migration of %s failed (%s)",          \
+                                frame->root->unique,                    \
+                                gf_fop_list[frame->root->op],           \
+                                state->resolve2.resolve_loc.path,       \
+                                strerror (op_errno));                   \
+                }                                                       \
+                                                                        \
+                if (op_ret < 0) {                                       \
+                        send_fuse_err (state->this, state->finh, op_errno); \
+                        free_fuse_state (state);                        \
+                        STACK_DESTROY (frame->root);                    \
+                } else {                                                \
+                        if (state->this->history)                       \
+                                gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: " \
+                                           "(%s)", frame->root->unique, \
+                                           gf_fop_list[frame->root->op], \
+                                           state->loc.path,             \
+                                           (state->fd == NULL)?         \
+                                           uuid_utoa (state->loc.gfid): \
+                                           uuid_utoa (state->fd->inode->gfid));\
+                        STACK_WIND (frame, ret, xl, xl->fops->fop, args); \
+                }                                                       \
+                                                                        \
+        } while (0)
+
+
+#define FUSE_FOP_COOKIE(state, xl, ret, cky, op_num, fop, args ...)     \
+        do {                                                            \
+                call_frame_t *frame  = NULL;                            \
+                xlator_t     *xl     = NULL;                            \
+                int32_t       op_ret = 0, op_errno = 0;                 \
+                                                                        \
+                frame = get_call_frame_for_req (state);                 \
+                if (!frame) {                                           \
+                        gf_log ("glusterfs-fuse",                       \
+                                GF_LOG_ERROR,                           \
+                                "FUSE message"                          \
+                                " unique %"PRIu64" opcode %d:"          \
+                                " frame allocation failed",             \
+                                state->finh->unique,                    \
+                                state->finh->opcode);                   \
+                        free_fuse_state (state);                        \
+                        return 0;                                       \
+                }                                                       \
+                                                                        \
+                frame->root->state = state;                             \
+                frame->root->op    = op_num;                            \
+		frame->op          = op_num;				\
+                                                                        \
+                xl = state->active_subvol;				\
+                if (!xl) {                                              \
+                        gf_log_callingfn ("glusterfs-fuse", GF_LOG_ERROR, \
+                                          "xl is NULL");                \
+                        op_errno = ENOENT;                              \
+                        op_ret = -1;                                    \
+                } else if (state->resolve.op_ret < 0) {                 \
+                        op_errno = state->resolve.op_errno;             \
+                        op_ret = -1;                                    \
+                        if (op_num == GF_FOP_LOOKUP) {                  \
+                                gf_log ("glusterfs-fuse",               \
+                                        (op_errno == ENOENT ? GF_LOG_TRACE \
+                                         : GF_LOG_WARNING),             \
+                                        "%"PRIu64": %s() %s => -1 (%s)", \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        state->resolve.resolve_loc.path, \
+                                        strerror (op_errno));           \
+                        } else {                                        \
+                                gf_log ("glusterfs-fuse",               \
+                                        GF_LOG_WARNING,                 \
+                                        "%"PRIu64": %s() inode "        \
+                                        "migration of %s failed (%s)",  \
+                                        frame->root->unique,            \
+                                        gf_fop_list[frame->root->op],   \
+                                        state->resolve.resolve_loc.path, \
+                                        strerror (op_errno));           \
+                        }                                               \
+                } else if (state->resolve2.op_ret < 0) {                \
+                        op_errno = state->resolve2.op_errno;            \
+                        op_ret = -1;                                    \
+                        gf_log ("glusterfs-fuse",                       \
+                                GF_LOG_WARNING,                         \
+                                "%"PRIu64": %s() inode "                \
+                                "migration of %s failed (%s)",          \
+                                frame->root->unique,                    \
+                                gf_fop_list[frame->root->op],           \
+                                state->resolve2.resolve_loc.path,       \
+                                strerror (op_errno));                   \
+                }                                                       \
+                                                                        \
+                if (op_ret < 0) {                                       \
+                        send_fuse_err (state->this, state->finh, op_errno); \
+                        free_fuse_state (state);                        \
+                        STACK_DESTROY (frame->root);                    \
+                } else {                                                \
+                        if (xl->history)                                \
+                                gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: " \
+                                           "(%s)", frame->root->unique, \
+                                           gf_fop_list[frame->root->op], \
+                                           state->loc.path,             \
+                                           uuid_utoa (state->loc.gfid)); \
+                        STACK_WIND_COOKIE (frame, ret, cky, xl, xl->fops->fop, \
+                                           args);                       \
+                }                                                       \
+        } while (0)
+
+#define GF_SELECT_LOG_LEVEL(_errno)                     \
+        (((_errno == ENOENT) || (_errno == ESTALE))?    \
+         GF_LOG_DEBUG)
+
+#define GET_STATE(this, finh, state)                                       \
+        do {                                                               \
+                state = get_fuse_state (this, finh);                       \
+                if (!state) {                                              \
+                        gf_log ("glusterfs-fuse",                          \
+                                GF_LOG_ERROR,                              \
+                                "FUSE message unique %"PRIu64" opcode %d:" \
+                                " state allocation failed",                \
+                                finh->unique, finh->opcode);               \
+                                                                           \
+                        send_fuse_err (this, finh, ENOMEM);                \
+                        GF_FREE (finh);                                    \
+                                                                           \
+                        return;                                            \
+                }                                                          \
+        } while (0)
+
+#define FUSE_ENTRY_CREATE(this, priv, finh, state, fci, op)             \
+                do {                                                    \
+                        if (priv->proto_minor >= 12)                    \
+                                state->mode &= ~fci->umask;             \
+                        if (priv->proto_minor >= 12 && priv->acl) {     \
+                                state->xdata = dict_new ();             \
+                                if (!state->xdata) {                    \
+                                        gf_log ("glusterfs-fuse",       \
+                                                GF_LOG_WARNING,         \
+                                                "%s failed to allocate " \
+                                                "a param dictionary", op); \
+                                        send_fuse_err (this, finh, ENOMEM); \
+                                        free_fuse_state (state);        \
+                                        return;                         \
+                                }                                       \
+                                state->umask = fci->umask;              \
+                                                                        \
+/* TODO: remove this after 3.4.0 release. keeping it for the            \
+   sake of backward compatibility with old (3.3.[01])                   \
+   releases till then. */                                               \
+                                ret = dict_set_int16 (state->xdata, "umask", \
+                                                      fci->umask);      \
+                                if (ret < 0) {                          \
+                                        gf_log ("glusterfs-fuse",       \
+                                                GF_LOG_WARNING,         \
+                                                "%s Failed adding umask"\
+                                                " to request", op);     \
+                                        dict_destroy (state->xdata);    \
+                                        send_fuse_err (this, finh, ENOMEM); \
+                                        free_fuse_state (state);        \
+                                        return;                         \
+                                }                                       \
+                                ret = dict_set_int16 (state->xdata, "mode", \
+                                                      fci->mode);       \
+                                if (ret < 0) {                          \
+                                        gf_log ("glusterfs-fuse",       \
+                                                GF_LOG_WARNING,         \
+                                                "%s Failed adding mode " \
+                                                "to request", op);         \
+                                        dict_destroy (state->xdata);    \
+                                        send_fuse_err (this, finh, ENOMEM); \
+                                        free_fuse_state (state);        \
+                                        return;                         \
+                                }                                       \
+                        }                                               \
+                } while (0)
+
+#define fuse_log_eh_fop(this, state, frame, op_ret, op_errno)               \
+        do {                                                            \
+                if (this->history) {                                    \
+                        if (state->fd)                                  \
+                                gf_log_eh ("op_ret: %d, op_errno: %d, " \
+                                           "%"PRIu64", %s () => %p, gfid: %s", \
+                                           op_ret, op_errno,            \
+                                           frame->root->unique,         \
+                                           gf_fop_list[frame->root->op], \
+                                           state->fd,                   \
+                                           uuid_utoa (state->fd->inode->gfid)); \
+                        else                                            \
+                                gf_log_eh ("op_ret: %d, op_errno: %d, " \
+                                           "%"PRIu64", %s () => %s, gfid: %s", \
+                                           op_ret, op_errno,            \
+                                           frame->root->unique,         \
+                                           gf_fop_list[frame->root->op], \
+                                           state->loc.path,             \
+                                           uuid_utoa (state->loc.gfid)); \
+                }                                                       \
+        } while(0)
+
+#define fuse_log_eh(this, args...)              \
+        do {                                    \
+                if (this->history)              \
+                        gf_log_eh(args);        \
+        } while (0)
+
+static inline xlator_t *
+fuse_active_subvol (xlator_t *fuse)
+{
+        fuse_private_t *priv = NULL;
+
+        priv = fuse->private;
+
+        return priv->active_subvol;
+}
+
+
+typedef enum {
+        RESOLVE_MUST = 1,
+        RESOLVE_NOT,
+        RESOLVE_MAY,
+        RESOLVE_DONTCARE,
+        RESOLVE_EXACT
+} fuse_resolve_type_t;
+
+
+typedef struct {
+        fuse_resolve_type_t    type;
+        fd_t                  *fd;
+        char                  *path;
+        char                  *bname;
+        u_char                 gfid[16];
+	inode_t               *hint;
+        u_char                 pargfid[16];
+	inode_t               *parhint;
+	char                  *resolved;
+        int                    op_ret;
+        int                    op_errno;
+        loc_t                  resolve_loc;
+} fuse_resolve_t;
+
+
+typedef struct {
+        void             *pool;
+        xlator_t         *this;
+	xlator_t         *active_subvol;
+        inode_table_t    *itable;
+        loc_t             loc;
+        loc_t             loc2;
+        fuse_in_header_t *finh;
+        int32_t           flags;
+        off_t             off;
+        size_t            size;
+        unsigned long     nlookup;
+        fd_t             *fd;
+        dict_t           *xattr;
+        dict_t           *xdata;
+        char             *name;
+        char              is_revalidate;
+        gf_boolean_t      truncate_needed;
+        gf_lock_t         lock;
+        uint64_t          lk_owner;
+
+        /* used within resolve_and_resume */
+        /* */
+        fuse_resolve_t resolve;
+        fuse_resolve_t resolve2;
+
+        loc_t        *loc_now;
+        fuse_resolve_t *resolve_now;
+
+        void *resume_fn;
+
+        int            valid;
+        int            mask;
+        dev_t          rdev;
+        mode_t         mode;
+        mode_t         umask;
+        struct iatt    attr;
+        struct gf_flock   lk_lock;
+        struct iovec   vector;
+
+        uuid_t         gfid;
+        uint32_t       io_flags;
+        int32_t        fd_no;
+} fuse_state_t;
+
+typedef struct {
+        uint32_t  open_flags;
+        char      migration_failed;
+        fd_t     *activefd;
+} fuse_fd_ctx_t;
+
+typedef void (*fuse_resume_fn_t) (fuse_state_t *state);
+
+GF_MUST_CHECK int32_t
+fuse_loc_fill (loc_t *loc, fuse_state_t *state, ino_t ino,
+               ino_t par, const char *name);
+call_frame_t *get_call_frame_for_req (fuse_state_t *state);
+fuse_state_t *get_fuse_state (xlator_t *this, fuse_in_header_t *finh);
+void free_fuse_state (fuse_state_t *state);
+void gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa,
+                        gf_boolean_t enable_ino32);
+void gf_fuse_fill_dirent (gf_dirent_t *entry, struct fuse_dirent *fde,
+                          gf_boolean_t enable_ino32);
+uint64_t inode_to_fuse_nodeid (inode_t *inode);
+xlator_t *fuse_active_subvol (xlator_t *fuse);
+inode_t *fuse_ino_to_inode (uint64_t ino, xlator_t *fuse);
+int send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error);
+int fuse_gfid_set (fuse_state_t *state);
+int fuse_flip_xattr_ns (struct fuse_private *priv, char *okey, char **nkey);
+fuse_fd_ctx_t * __fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
+fuse_fd_ctx_t * fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
+
+int fuse_resolve_and_resume (fuse_state_t *state, fuse_resume_fn_t fn);
+int fuse_resolve_inode_init (fuse_state_t *state, fuse_resolve_t *resolve,
+			     ino_t ino);
+int fuse_resolve_entry_init (fuse_state_t *state, fuse_resolve_t *resolve,
+			     ino_t par, char *name);
+int fuse_resolve_fd_init (fuse_state_t *state, fuse_resolve_t *resolve,
+			  fd_t *fd);
+int fuse_ignore_xattr_set (fuse_private_t *priv, char *key);
+int dump_history_fuse (circular_buffer_t *cb, void *data);
+#endif /* _GF_FUSE_BRIDGE_H_ */
diff --git a/xlators/mount/fuse/src/fuse-extra.c b/xlators/mount/fuse/src/fuse-extra.c
deleted file mode 100644
index 93574d174..000000000
--- a/xlators/mount/fuse/src/fuse-extra.c
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
-   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif /* _CONFIG_H */
-
-#include "fuse-extra.h"
-#include "common-utils.h"
-#include <stdio.h>
-#include <pthread.h>
-#include <stdlib.h>
-#include <string.h>
-#include "common-utils.h"
-
-struct fuse_req;
-struct fuse_ll;
-
-struct fuse_req {
-    struct fuse_ll *f;
-    uint64_t unique;
-    int ctr;
-    pthread_mutex_t lock;
-    struct fuse_ctx ctx;
-    struct fuse_chan *ch;
-    int interrupted;
-    union {
-        struct {
-            uint64_t unique;
-        } i;
-        struct {
-            fuse_interrupt_func_t func;
-            void *data;
-        } ni;
-    } u;
-    struct fuse_req *next;
-    struct fuse_req *prev;
-};
-
-struct fuse_ll {
-    int debug;
-    int allow_root;
-    struct fuse_lowlevel_ops op;
-    int got_init;
-    void *userdata;
-    uid_t owner;
-    struct fuse_conn_info conn;
-    struct fuse_req list;
-    struct fuse_req interrupts;
-    pthread_mutex_t lock;
-    int got_destroy;
-};
-
-struct fuse_out_header {
-  uint32_t   len;
-  int32_t    error;
-  uint64_t   unique;
-};
-
-uint64_t req_callid (fuse_req_t req)
-{
-  return req->unique;
-}
-
-static void destroy_req(fuse_req_t req)
-{
-    pthread_mutex_destroy (&req->lock);
-    FREE (req);
-}
-
-static void list_del_req(struct fuse_req *req)
-{
-    struct fuse_req *prev = req->prev;
-    struct fuse_req *next = req->next;
-    prev->next = next;
-    next->prev = prev;
-}
-
-static void
-free_req (fuse_req_t req)
-{
-  int ctr;
-  struct fuse_ll *f = req->f;
-  
-  pthread_mutex_lock(&req->lock);
-  req->u.ni.func = NULL;
-  req->u.ni.data = NULL;
-  pthread_mutex_unlock(&req->lock);
-
-  pthread_mutex_lock(&f->lock);
-  list_del_req(req);
-  ctr = --req->ctr;
-  pthread_mutex_unlock(&f->lock);
-  if (!ctr)
-    destroy_req(req);
-}
-
-int32_t
-fuse_reply_vec (fuse_req_t req,
-		struct iovec *vector,
-		int32_t count)
-{
-  int32_t error = 0;
-  struct fuse_out_header out;
-  struct iovec *iov;
-  int res;
-
-  iov = alloca ((count + 1) * sizeof (*vector));
-  out.unique = req->unique;
-  out.error = error;
-  iov[0].iov_base = &out;
-  iov[0].iov_len = sizeof(struct fuse_out_header);
-  memcpy (&iov[1], vector, count * sizeof (*vector));
-  count++;
-  out.len = iov_length(iov, count);
-  res = fuse_chan_send(req->ch, iov, count);
-  free_req(req);
-
-  return res;
-}
diff --git a/xlators/mount/fuse/src/fuse-extra.h b/xlators/mount/fuse/src/fuse-extra.h
deleted file mode 100644
index 0e8052b5a..000000000
--- a/xlators/mount/fuse/src/fuse-extra.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
-   Copyright (c) 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _FUSE_EXTRA_H
-#define _FUSE_EXTRA_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif /* _CONFIG_H */
-
-#include <stdlib.h>
-#include <fuse/fuse_lowlevel.h>
-
-#define GLUSTERFS_XATTR_LEN_MAX  65536
-
-uint64_t req_callid (fuse_req_t req);
-
-size_t fuse_dirent_size (size_t dname_len);
-
-int32_t
-fuse_reply_vec (fuse_req_t req,
-		struct iovec *vector,
-		int32_t count);
-
-#endif /* _FUSE_EXTRA_H */
diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c
new file mode 100644
index 000000000..4d478b919
--- /dev/null
+++ b/xlators/mount/fuse/src/fuse-helpers.c
@@ -0,0 +1,605 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifdef __NetBSD__
+#define _KMEMUSER
+#endif
+
+#include "fuse-bridge.h"
+#if defined(GF_SOLARIS_HOST_OS)
+#include <sys/procfs.h>
+#else
+#include <sys/sysctl.h>
+#endif
+
+
+static void
+fuse_resolve_wipe (fuse_resolve_t *resolve)
+{
+        GF_FREE ((void *)resolve->path);
+
+        GF_FREE ((void *)resolve->bname);
+
+        GF_FREE ((void *)resolve->resolved);
+
+        if (resolve->fd)
+                fd_unref (resolve->fd);
+
+        loc_wipe (&resolve->resolve_loc);
+
+	if (resolve->hint) {
+		inode_unref (resolve->hint);
+		resolve->hint = 0;
+	}
+
+	if (resolve->parhint) {
+		inode_unref (resolve->parhint);
+		resolve->parhint = 0;
+	}
+}
+
+
+void
+free_fuse_state (fuse_state_t *state)
+{
+        xlator_t       *this     = NULL;
+        fuse_private_t *priv     = NULL;
+        uint64_t        winds    = 0;
+        char            switched = 0;
+
+        this = state->this;
+
+        priv = this->private;
+
+        loc_wipe (&state->loc);
+
+        loc_wipe (&state->loc2);
+
+        if (state->xdata) {
+                dict_unref (state->xdata);
+                state->xdata = (void *)0xaaaaeeee;
+        }
+        if (state->xattr)
+                dict_unref (state->xattr);
+
+        if (state->name) {
+                GF_FREE (state->name);
+                state->name = NULL;
+        }
+        if (state->fd) {
+                fd_unref (state->fd);
+                state->fd = (void *)0xfdfdfdfd;
+        }
+        if (state->finh) {
+                GF_FREE (state->finh);
+                state->finh = NULL;
+        }
+
+        fuse_resolve_wipe (&state->resolve);
+        fuse_resolve_wipe (&state->resolve2);
+
+        pthread_mutex_lock (&priv->sync_mutex);
+        {
+                winds = --state->active_subvol->winds;
+                switched = state->active_subvol->switched;
+        }
+        pthread_mutex_unlock (&priv->sync_mutex);
+
+        if ((winds == 0) && (switched)) {
+                xlator_notify (state->active_subvol, GF_EVENT_PARENT_DOWN,
+                               state->active_subvol, NULL);
+        }
+
+#ifdef DEBUG
+        memset (state, 0x90, sizeof (*state));
+#endif
+        GF_FREE (state);
+        state = NULL;
+}
+
+
+fuse_state_t *
+get_fuse_state (xlator_t *this, fuse_in_header_t *finh)
+{
+        fuse_state_t   *state         = NULL;
+	xlator_t       *active_subvol = NULL;
+        fuse_private_t *priv          = NULL;
+
+        state = (void *)GF_CALLOC (1, sizeof (*state),
+                                   gf_fuse_mt_fuse_state_t);
+        if (!state)
+                return NULL;
+
+	state->this = THIS;
+        priv = this->private;
+
+        pthread_mutex_lock (&priv->sync_mutex);
+        {
+                active_subvol = fuse_active_subvol (state->this);
+                active_subvol->winds++;
+        }
+        pthread_mutex_unlock (&priv->sync_mutex);
+
+	state->active_subvol = active_subvol;
+	state->itable = active_subvol->itable;
+
+        state->pool = this->ctx->pool;
+        state->finh = finh;
+        state->this = this;
+
+        LOCK_INIT (&state->lock);
+
+        return state;
+}
+
+
+#define FUSE_MAX_AUX_GROUPS 32 /* We can get only up to 32 aux groups from /proc */
+void
+frame_fill_groups (call_frame_t *frame)
+{
+#if defined(GF_LINUX_HOST_OS)
+        char         filename[32];
+        char         line[4096];
+        char        *ptr = NULL;
+        FILE        *fp = NULL;
+        int          idx = 0;
+        long int     id = 0;
+        char        *saveptr = NULL;
+        char        *endptr = NULL;
+        int          ret = 0;
+
+        ret = snprintf (filename, sizeof filename, "/proc/%d/status", frame->root->pid);
+        if (ret >= sizeof filename)
+                goto out;
+
+        fp = fopen (filename, "r");
+        if (!fp)
+                goto out;
+
+	if (call_stack_alloc_groups (frame->root, FUSE_MAX_AUX_GROUPS) != 0)
+		goto out;
+
+        while ((ptr = fgets (line, sizeof line, fp))) {
+                if (strncmp (ptr, "Groups:", 7) != 0)
+                        continue;
+
+                ptr = line + 8;
+
+                for (ptr = strtok_r (ptr, " \t\r\n", &saveptr);
+                     ptr;
+                     ptr = strtok_r (NULL, " \t\r\n", &saveptr)) {
+                        errno = 0;
+                        id = strtol (ptr, &endptr, 0);
+                        if (errno == ERANGE)
+                                break;
+                        if (!endptr || *endptr)
+                                break;
+                        frame->root->groups[idx++] = id;
+                        if (idx == FUSE_MAX_AUX_GROUPS)
+                                break;
+                }
+
+                frame->root->ngrps = idx;
+                break;
+        }
+out:
+        if (fp)
+                fclose (fp);
+#elif defined(GF_SOLARIS_HOST_OS)
+        char         filename[32];
+        char         scratch[128];
+        prcred_t    *prcred = (prcred_t *) scratch;
+        FILE        *fp = NULL;
+        int          ret = 0;
+	int          ngrps;
+
+        ret = snprintf (filename, sizeof filename,
+                        "/proc/%d/cred", frame->root->pid);
+
+        if (ret < sizeof filename) {
+                fp = fopen (filename, "r");
+                if (fp != NULL) {
+                        if (fgets (scratch, sizeof scratch, fp) != NULL) {
+                                ngrps = MIN(prcred->pr_ngroups,
+					    GF_MAX_AUX_GROUPS);
+				if (call_stack_alloc_groups (frame->root,
+							     ngrps) != 0)
+					return;
+                        }
+                        fclose (fp);
+                 }
+         }
+#elif defined(CTL_KERN) /* DARWIN and *BSD */
+        /* 
+           N.B. CTL_KERN is an enum on Linux. (Meaning, if it's not
+           obvious, that it's not subject to preprocessor directives 
+           like '#if defined'.)
+           Unlike Linux, on Mac OS and the BSDs it is a #define. We
+           could test to see that KERN_PROC is defined, but, barring any 
+           evidence to the contrary, I think that's overkill.
+           We might also test that GF_DARWIN_HOST_OS is defined, why
+           limit this to just Mac OS. It's equally valid for the BSDs
+           and we do have people building on NetBSD and FreeBSD.
+        */
+        int name[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, frame->root->pid };
+        size_t namelen = sizeof name / sizeof name[0];
+        struct kinfo_proc kp;
+        size_t kplen = sizeof(kp);
+        int i, ngroups;
+
+        if (sysctl(name, namelen, &kp, &kplen, NULL, 0) != 0)
+                return;
+        ngroups = MIN(kp.kp_eproc.e_ucred.cr_ngroups, GF_MAX_AUX_GROUPS);
+	if (call_stack_alloc_groups (frame->root, ngroups) != 0)
+		return;
+        for (i = 0; i < ngroups; i++)
+                frame->root->groups[i] = kp.kp_eproc.e_ucred.cr_groups[i];
+        frame->root->ngrps = ngroups;
+#else
+        frame->root->ngrps = 0;
+#endif /* GF_LINUX_HOST_OS */
+}
+
+/*
+ * Get the groups for the PID associated with this frame. If enabled,
+ * use the gid cache to reduce group list collection.
+ */
+static void get_groups(fuse_private_t *priv, call_frame_t *frame)
+{
+	int i;
+	const gid_list_t *gl;
+	gid_list_t agl;
+
+        if (-1 == priv->gid_cache_timeout) {
+                frame->root->ngrps = 0;
+                return;
+        }
+
+	if (!priv->gid_cache_timeout) {
+		frame_fill_groups(frame);
+		return;
+	}
+
+	gl = gid_cache_lookup(&priv->gid_cache, frame->root->pid);
+	if (gl) {
+		if (call_stack_alloc_groups (frame->root, gl->gl_count) != 0)
+			return;
+		frame->root->ngrps = gl->gl_count;
+		for (i = 0; i < gl->gl_count; i++)
+			frame->root->groups[i] = gl->gl_list[i];
+		gid_cache_release(&priv->gid_cache, gl);
+		return;
+	}
+
+	frame_fill_groups (frame);
+
+	agl.gl_id = frame->root->pid;
+	agl.gl_count = frame->root->ngrps;
+	agl.gl_list = GF_CALLOC(frame->root->ngrps, sizeof(gid_t),
+			gf_fuse_mt_gids_t);
+	if (!agl.gl_list)
+		return;
+
+	for (i = 0; i < frame->root->ngrps; i++)
+		agl.gl_list[i] = frame->root->groups[i];
+
+	if (gid_cache_add(&priv->gid_cache, &agl) != 1)
+		GF_FREE(agl.gl_list);
+}
+
+call_frame_t *
+get_call_frame_for_req (fuse_state_t *state)
+{
+        call_pool_t           *pool = NULL;
+        fuse_in_header_t      *finh = NULL;
+        call_frame_t          *frame = NULL;
+        xlator_t              *this = NULL;
+        fuse_private_t        *priv = NULL;
+
+        pool = state->pool;
+        finh = state->finh;
+        this = state->this;
+        priv = this->private;
+
+        frame = create_frame (this, pool);
+        if (!frame)
+                return NULL;
+
+        if (finh) {
+                frame->root->uid      = finh->uid;
+                frame->root->gid      = finh->gid;
+                frame->root->pid      = finh->pid;
+                frame->root->unique   = finh->unique;
+                set_lk_owner_from_uint64 (&frame->root->lk_owner,
+                                          state->lk_owner);
+        }
+
+	get_groups(priv, frame);
+
+        if (priv && priv->client_pid_set)
+                frame->root->pid = priv->client_pid;
+
+        frame->root->type = GF_OP_TYPE_FOP;
+
+        return frame;
+}
+
+
+inode_t *
+fuse_ino_to_inode (uint64_t ino, xlator_t *fuse)
+{
+        inode_t  *inode = NULL;
+        xlator_t *active_subvol = NULL;
+
+        if (ino == 1) {
+                active_subvol = fuse_active_subvol (fuse);
+                if (active_subvol)
+                        inode = active_subvol->itable->root;
+        } else {
+                inode = (inode_t *) (unsigned long) ino;
+                inode_ref (inode);
+        }
+
+        return inode;
+}
+
+uint64_t
+inode_to_fuse_nodeid (inode_t *inode)
+{
+        if (!inode)
+		return 0;
+	if (__is_root_gfid (inode->gfid))
+                return 1;
+
+        return (unsigned long) inode;
+}
+
+
+GF_MUST_CHECK int32_t
+fuse_loc_fill (loc_t *loc, fuse_state_t *state, ino_t ino,
+               ino_t par, const char *name)
+{
+        inode_t  *inode = NULL;
+        inode_t  *parent = NULL;
+        int32_t   ret = -1;
+        char     *path = NULL;
+        uuid_t    null_gfid = {0,};
+
+        /* resistance against multiple invocation of loc_fill not to get
+           reference leaks via inode_search() */
+
+        if (name) {
+                parent = loc->parent;
+                if (!parent) {
+                        parent = fuse_ino_to_inode (par, state->this);
+                        loc->parent = parent;
+                        if (parent)
+                                uuid_copy (loc->pargfid, parent->gfid);
+                }
+
+                inode = loc->inode;
+                if (!inode) {
+                        inode = inode_grep (parent->table, parent, name);
+                        loc->inode = inode;
+                }
+
+                ret = inode_path (parent, name, &path);
+                if (ret <= 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+                                "inode_path failed for %s/%s",
+                                (parent)?uuid_utoa (parent->gfid):"0", name);
+                        goto fail;
+                }
+                loc->path = path;
+        } else {
+                inode = loc->inode;
+                if (!inode) {
+                        inode = fuse_ino_to_inode (ino, state->this);
+                        loc->inode = inode;
+                        if (inode)
+                                uuid_copy (loc->gfid, inode->gfid);
+                }
+
+                parent = loc->parent;
+                if (!parent) {
+                        parent = inode_parent (inode, null_gfid, NULL);
+                        loc->parent = parent;
+                        if (parent)
+                                uuid_copy (loc->pargfid, parent->gfid);
+
+                }
+
+                ret = inode_path (inode, NULL, &path);
+                if (ret <= 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
+                                "inode_path failed for %s",
+                                (inode) ? uuid_utoa (inode->gfid) : "0");
+                        goto fail;
+                }
+                loc->path = path;
+        }
+
+        if (loc->path) {
+                loc->name = strrchr (loc->path, '/');
+                if (loc->name)
+                        loc->name++;
+                else
+                        loc->name = "";
+        }
+
+        if ((ino != 1) && (parent == NULL)) {
+                gf_log ("fuse-bridge", GF_LOG_DEBUG,
+                        "failed to search parent for %"PRId64"/%s (%"PRId64")",
+                        (ino_t)par, name, (ino_t)ino);
+                ret = -1;
+                goto fail;
+        }
+        ret = 0;
+fail:
+        /* this should not happen as inode_path returns -1 when buf is NULL
+           for sure */
+        if (path && !loc->path)
+                GF_FREE (path);
+        return ret;
+}
+
+/* Use the same logic as the Linux NFS-client */
+#define GF_FUSE_SQUASH_INO(ino) ((uint32_t) ino) ^ (ino >> 32)
+
+/* courtesy of folly */
+void
+gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa, gf_boolean_t enable_ino32)
+{
+        if (enable_ino32)
+                fa->ino = GF_FUSE_SQUASH_INO(st->ia_ino);
+        else
+                fa->ino = st->ia_ino;
+
+        fa->size        = st->ia_size;
+        fa->blocks      = st->ia_blocks;
+        fa->atime       = st->ia_atime;
+        fa->mtime       = st->ia_mtime;
+        fa->ctime       = st->ia_ctime;
+        fa->atimensec   = st->ia_atime_nsec;
+        fa->mtimensec   = st->ia_mtime_nsec;
+        fa->ctimensec   = st->ia_ctime_nsec;
+        fa->mode        = st_mode_from_ia (st->ia_prot, st->ia_type);
+        fa->nlink       = st->ia_nlink;
+        fa->uid         = st->ia_uid;
+        fa->gid         = st->ia_gid;
+        fa->rdev        = makedev (ia_major (st->ia_rdev),
+                                   ia_minor (st->ia_rdev));
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+        fa->blksize     = st->ia_blksize;
+#endif
+#ifdef GF_DARWIN_HOST_OS
+        fa->crtime      = (uint64_t)-1;
+        fa->crtimensec  = (uint32_t)-1;
+        fa->flags       = 0;
+#endif
+}
+
+void
+gf_fuse_fill_dirent (gf_dirent_t *entry, struct fuse_dirent *fde, gf_boolean_t enable_ino32)
+{
+        if (enable_ino32)
+                fde->ino = GF_FUSE_SQUASH_INO(entry->d_ino);
+        else
+                fde->ino = entry->d_ino;
+
+        fde->off         = entry->d_off;
+        fde->type        = entry->d_type;
+        fde->namelen     = strlen (entry->d_name);
+        strncpy (fde->name, entry->d_name, fde->namelen);
+}
+
+static int
+fuse_do_flip_xattr_ns (char *okey, const char *nns, char **nkey)
+{
+        int   ret = 0;
+        char *key = NULL;
+
+        okey = strchr (okey, '.');
+        GF_ASSERT (okey);
+
+        key = GF_CALLOC (1, strlen (nns) + strlen(okey) + 1,
+                         gf_common_mt_char);
+        if (!key) {
+                ret = -1;
+                goto out;
+        }
+
+        strcpy (key, nns);
+        strcat (key, okey);
+
+        *nkey = key;
+
+ out:
+        return ret;
+}
+
+static int
+fuse_xattr_alloc_default (char *okey, char **nkey)
+{
+        int ret = 0;
+
+        *nkey = gf_strdup (okey);
+        if (!*nkey)
+                ret = -1;
+        return ret;
+}
+
+#define PRIV_XA_NS   "trusted"
+#define UNPRIV_XA_NS "system"
+
+int
+fuse_flip_xattr_ns (fuse_private_t *priv, char *okey, char **nkey)
+{
+        int             ret       = 0;
+        gf_boolean_t    need_flip = _gf_false;
+
+        switch (priv->client_pid) {
+        case GF_CLIENT_PID_GSYNCD:
+                /* valid xattr(s): *xtime, volume-mark* */
+                gf_log("glusterfs-fuse", GF_LOG_DEBUG, "PID: %d, checking xattr(s): "
+                       "volume-mark*, *xtime", priv->client_pid);
+                if ( (strcmp (okey, UNPRIV_XA_NS".glusterfs.volume-mark") == 0)
+                     || (fnmatch (UNPRIV_XA_NS".glusterfs.volume-mark.*", okey, FNM_PERIOD) == 0)
+                     || (fnmatch (UNPRIV_XA_NS".glusterfs.*.xtime", okey, FNM_PERIOD) == 0) )
+                        need_flip = _gf_true;
+                break;
+
+        case GF_CLIENT_PID_HADOOP:
+                /* valid xattr(s): pathinfo */
+                gf_log("glusterfs-fuse", GF_LOG_DEBUG, "PID: %d, checking xattr(s): "
+                       "pathinfo", priv->client_pid);
+                if (strcmp (okey, UNPRIV_XA_NS".glusterfs.pathinfo") == 0)
+                        need_flip = _gf_true;
+                break;
+        }
+
+        if (need_flip) {
+                gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "flipping %s to "PRIV_XA_NS" equivalent",
+                        okey);
+                ret = fuse_do_flip_xattr_ns (okey, PRIV_XA_NS, nkey);
+        } else {
+                /* if we cannot match, continue with what we got */
+                ret = fuse_xattr_alloc_default (okey, nkey);
+        }
+
+        return ret;
+}
+
+int
+fuse_ignore_xattr_set (fuse_private_t *priv, char *key)
+{
+        int ret = 0;
+
+        /* don't mess with user namespace */
+        if (fnmatch ("user.*", key, FNM_PERIOD) == 0)
+                goto out;
+
+        if (priv->client_pid != GF_CLIENT_PID_GSYNCD)
+                goto out;
+
+        /* trusted NS check */
+        if (!((fnmatch ("*.glusterfs.*.xtime", key, FNM_PERIOD) == 0)
+              || (fnmatch ("*.glusterfs.volume-mark",
+                           key, FNM_PERIOD) == 0)
+              || (fnmatch ("*.glusterfs.volume-mark.*",
+                           key, FNM_PERIOD) == 0)))
+                ret = -1;
+
+ out:
+        gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "%s setxattr: key [%s], "
+                " client pid [%d]", (ret ? "disallowing" : "allowing"), key,
+                priv->client_pid);
+
+        return ret;
+}
diff --git a/xlators/mount/fuse/src/fuse-mem-types.h b/xlators/mount/fuse/src/fuse-mem-types.h
new file mode 100644
index 000000000..28b4dfbdd
--- /dev/null
+++ b/xlators/mount/fuse/src/fuse-mem-types.h
@@ -0,0 +1,28 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __FUSE_MEM_TYPES_H__
+#define __FUSE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_fuse_mem_types_ {
+        gf_fuse_mt_iovec  = gf_common_mt_end + 1,
+        gf_fuse_mt_fuse_private_t,
+        gf_fuse_mt_char,
+        gf_fuse_mt_iov_base,
+        gf_fuse_mt_fuse_state_t,
+        gf_fuse_mt_fd_ctx_t,
+        gf_fuse_mt_graph_switch_args_t,
+	gf_fuse_mt_gids_t,
+        gf_fuse_mt_end
+};
+#endif
+
diff --git a/xlators/mount/fuse/src/fuse-resolve.c b/xlators/mount/fuse/src/fuse-resolve.c
new file mode 100644
index 000000000..8565ce0e4
--- /dev/null
+++ b/xlators/mount/fuse/src/fuse-resolve.c
@@ -0,0 +1,724 @@
+/*
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "fuse-bridge.h"
+
+static int
+fuse_resolve_all (fuse_state_t *state);
+
+int fuse_resolve_continue (fuse_state_t *state);
+int fuse_resolve_entry_simple (fuse_state_t *state);
+int fuse_resolve_inode_simple (fuse_state_t *state);
+int fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
+                     xlator_t *new_subvol);
+
+fuse_fd_ctx_t *
+fuse_fd_ctx_get (xlator_t *this, fd_t *fd);
+
+gf_boolean_t fuse_inode_needs_lookup (inode_t *inode, xlator_t *this);
+
+static int
+fuse_resolve_loc_touchup (fuse_state_t *state)
+{
+        fuse_resolve_t *resolve = NULL;
+        loc_t          *loc     = NULL;
+        char           *path    = NULL;
+        int             ret     = 0;
+
+        resolve = state->resolve_now;
+        loc     = state->loc_now;
+
+        if (!loc->path) {
+                if (loc->parent && resolve->bname) {
+                        ret = inode_path (loc->parent, resolve->bname, &path);
+			uuid_copy (loc->pargfid, loc->parent->gfid);
+			loc->name = resolve->bname;
+                } else if (loc->inode) {
+                        ret = inode_path (loc->inode, NULL, &path);
+			uuid_copy (loc->gfid, loc->inode->gfid);
+                }
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_TRACE,
+                                "return value inode_path %d", ret);
+                loc->path = path;
+        }
+
+        return 0;
+}
+
+
+int
+fuse_resolve_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			int op_ret, int op_errno, inode_t *inode,
+			struct iatt *buf, dict_t *xattr,
+			struct iatt *postparent)
+{
+        fuse_state_t   *state      = NULL;
+        fuse_resolve_t *resolve    = NULL;
+        inode_t        *link_inode = NULL;
+        loc_t          *resolve_loc   = NULL;
+
+        state = frame->root->state;
+        resolve = state->resolve_now;
+        resolve_loc = &resolve->resolve_loc;
+
+        STACK_DESTROY (frame->root);
+
+        if (op_ret == -1) {
+                gf_log (this->name, (op_errno == ENOENT)
+                        ? GF_LOG_DEBUG : GF_LOG_WARNING,
+                        "%s/%s: failed to resolve (%s)",
+                        uuid_utoa (resolve_loc->pargfid), resolve_loc->name,
+                        strerror (op_errno));
+                resolve->op_ret = -1;
+                resolve->op_errno = op_errno;
+                goto out;
+        }
+
+        link_inode = inode_link (inode, resolve_loc->parent,
+                                 resolve_loc->name, buf);
+
+	state->loc_now->inode = link_inode;
+
+out:
+        loc_wipe (resolve_loc);
+
+        fuse_resolve_continue (state);
+        return 0;
+}
+
+
+int
+fuse_resolve_entry (fuse_state_t *state)
+{
+	fuse_resolve_t   *resolve = NULL;
+	loc_t            *resolve_loc = NULL;
+
+	resolve = state->resolve_now;
+	resolve_loc = &resolve->resolve_loc;
+
+	resolve_loc->parent = inode_ref (state->loc_now->parent);
+	uuid_copy (resolve_loc->pargfid, state->loc_now->pargfid);
+        resolve_loc->name = resolve->bname;
+        resolve_loc->inode = inode_new (state->itable);
+
+        inode_path (resolve_loc->parent, resolve_loc->name,
+                    (char **) &resolve_loc->path);
+
+        FUSE_FOP (state, fuse_resolve_entry_cbk, GF_FOP_LOOKUP,
+                  lookup, resolve_loc, NULL);
+
+	return 0;
+}
+
+
+int
+fuse_resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, inode_t *inode,
+                       struct iatt *buf, dict_t *xattr, struct iatt *postparent)
+{
+        fuse_state_t   *state      = NULL;
+        fuse_resolve_t *resolve    = NULL;
+        inode_t        *link_inode = NULL;
+        loc_t          *loc_now   = NULL;
+
+        state = frame->root->state;
+        resolve = state->resolve_now;
+	loc_now = state->loc_now;
+
+        STACK_DESTROY (frame->root);
+
+        if (op_ret == -1) {
+                gf_log (this->name, (op_errno == ENOENT)
+                        ? GF_LOG_DEBUG : GF_LOG_WARNING,
+                        "%s: failed to resolve (%s)",
+                        uuid_utoa (resolve->resolve_loc.gfid),
+			strerror (op_errno));
+                loc_wipe (&resolve->resolve_loc);
+
+                /* resolve->op_ret can have 3 values: 0, -1, -2.
+                 * 0 : resolution was successful.
+                 * -1: parent inode could not be resolved.
+                 * -2: entry (inode corresponding to path) could not be resolved
+                 */
+
+                if (uuid_is_null (resolve->gfid)) {
+                        resolve->op_ret = -1;
+                } else {
+                        resolve->op_ret = -2;
+                }
+
+                resolve->op_errno = op_errno;
+                goto out;
+        }
+
+        loc_wipe (&resolve->resolve_loc);
+
+        link_inode = inode_link (inode, NULL, NULL, buf);
+
+        if (!link_inode)
+                goto out;
+
+	if (!uuid_is_null (resolve->gfid)) {
+		loc_now->inode = link_inode;
+		goto out;
+	}
+
+	loc_now->parent = link_inode;
+        uuid_copy (loc_now->pargfid, link_inode->gfid);
+
+	fuse_resolve_entry (state);
+
+        return 0;
+out:
+        fuse_resolve_continue (state);
+        return 0;
+}
+
+
+int
+fuse_resolve_gfid (fuse_state_t *state)
+{
+        fuse_resolve_t *resolve  = NULL;
+        loc_t          *resolve_loc = NULL;
+        int             ret      = 0;
+
+        resolve = state->resolve_now;
+        resolve_loc = &resolve->resolve_loc;
+
+        if (!uuid_is_null (resolve->pargfid)) {
+                uuid_copy (resolve_loc->gfid, resolve->pargfid);
+        } else if (!uuid_is_null (resolve->gfid)) {
+                uuid_copy (resolve_loc->gfid, resolve->gfid);
+        }
+
+	/* inode may already exist in case we are looking up an inode which was
+	   linked through readdirplus */
+	resolve_loc->inode = inode_find (state->itable, resolve_loc->gfid);
+	if (!resolve_loc->inode)
+		resolve_loc->inode = inode_new (state->itable);
+	ret = loc_path (resolve_loc, NULL);
+
+        if (ret <= 0) {
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to get the path for inode %s",
+                        uuid_utoa (resolve->gfid));
+        }
+
+        FUSE_FOP (state, fuse_resolve_gfid_cbk, GF_FOP_LOOKUP,
+                  lookup, resolve_loc, NULL);
+
+        return 0;
+}
+
+
+/*
+ * Return value:
+ * 0 - resolved parent and entry (as necessary)
+ * -1 - resolved parent but not entry (though necessary)
+ * 1 - resolved neither parent nor entry
+ */
+
+int
+fuse_resolve_parent_simple (fuse_state_t *state)
+{
+        fuse_resolve_t *resolve   = NULL;
+	loc_t          *loc       = NULL;
+        inode_t        *parent    = NULL;
+        inode_t        *inode     = NULL;
+
+        resolve = state->resolve_now;
+	loc = state->loc_now;
+
+	loc->name = resolve->bname;
+
+	parent = resolve->parhint;
+	if (parent->table == state->itable) {
+		if (fuse_inode_needs_lookup (parent, THIS))
+			return 1;
+
+		/* no graph switches since */
+		loc->parent = inode_ref (parent);
+		uuid_copy (loc->pargfid, parent->gfid);
+		loc->inode = inode_grep (state->itable, parent, loc->name);
+
+                /* nodeid for root is 1 and we blindly take the latest graph's
+                 * table->root as the parhint and because of this there is
+                 * ambiguity whether the entry should have existed or not, and
+                 * we took the conservative approach of assuming entry should
+                 * have been there even though it need not have (bug #804592).
+                 */
+                if ((loc->inode == NULL)
+                    && __is_root_gfid (parent->gfid)) {
+                        /* non decisive result - entry missing */
+                        return -1;
+                }
+
+		/* decisive result - resolution success */
+		return 0;
+	}
+
+        parent = inode_find (state->itable, resolve->pargfid);
+	if (!parent) {
+		/* non decisive result - parent missing */
+		return 1;
+	}
+	if (fuse_inode_needs_lookup (parent, THIS)) {
+		inode_unref (parent);
+		return 1;
+	}
+
+	loc->parent = parent;
+        uuid_copy (loc->pargfid, resolve->pargfid);
+
+	inode = inode_grep (state->itable, parent, loc->name);
+	if (inode) {
+		loc->inode = inode;
+		/* decisive result - resolution success */
+		return 0;
+	}
+
+	/* non decisive result - entry missing */
+        return -1;
+}
+
+
+int
+fuse_resolve_parent (fuse_state_t *state)
+{
+        int    ret = 0;
+
+        ret = fuse_resolve_parent_simple (state);
+        if (ret > 0) {
+                fuse_resolve_gfid (state);
+                return 0;
+        }
+
+	if (ret < 0) {
+		fuse_resolve_entry (state);
+		return 0;
+	}
+
+        fuse_resolve_continue (state);
+
+        return 0;
+}
+
+
+int
+fuse_resolve_inode_simple (fuse_state_t *state)
+{
+        fuse_resolve_t *resolve   = NULL;
+	loc_t          *loc = NULL;
+        inode_t        *inode     = NULL;
+
+        resolve = state->resolve_now;
+	loc = state->loc_now;
+
+	inode = resolve->hint;
+	if (inode->table == state->itable)
+		inode_ref (inode);
+	else
+		inode = inode_find (state->itable, resolve->gfid);
+
+        if (inode) {
+		if (!fuse_inode_needs_lookup (inode, THIS))
+			goto found;
+		/* inode was linked through readdirplus */
+		inode_unref (inode);
+	}
+
+        return 1;
+found:
+	loc->inode = inode;
+	return 0;
+}
+
+
+int
+fuse_resolve_inode (fuse_state_t *state)
+{
+        int                 ret = 0;
+
+        ret = fuse_resolve_inode_simple (state);
+
+        if (ret > 0) {
+                fuse_resolve_gfid (state);
+                return 0;
+        }
+
+        fuse_resolve_continue (state);
+
+        return 0;
+}
+
+
+int
+fuse_migrate_fd_task (void *data)
+{
+        int            ret        = -1;
+        fuse_state_t  *state      = NULL;
+        fd_t          *basefd     = NULL, *oldfd = NULL;
+        fuse_fd_ctx_t *basefd_ctx = NULL;
+        xlator_t      *old_subvol = NULL;
+
+        state = data;
+        if (state == NULL) {
+                goto out;
+        }
+
+        basefd = state->fd;
+
+        basefd_ctx = fuse_fd_ctx_get (state->this, basefd);
+
+        LOCK (&basefd->lock);
+        {
+                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd : basefd;
+                fd_ref (oldfd);
+        }
+        UNLOCK (&basefd->lock);
+
+        old_subvol = oldfd->inode->table->xl;
+
+        ret = fuse_migrate_fd (state->this, basefd, old_subvol,
+                               state->active_subvol);
+
+        LOCK (&basefd->lock);
+        {
+                if (ret < 0) {
+                        basefd_ctx->migration_failed = 1;
+                } else {
+                        basefd_ctx->migration_failed = 0;
+                }
+        }
+        UNLOCK (&basefd->lock);
+
+        ret = 0;
+
+out:
+        if (oldfd)
+                fd_unref (oldfd);
+
+        return ret;
+}
+
+
+static inline int
+fuse_migrate_fd_error (xlator_t *this, fd_t *fd)
+{
+        fuse_fd_ctx_t *fdctx = NULL;
+        char           error = 0;
+
+        fdctx = fuse_fd_ctx_get (this, fd);
+        if (fdctx != NULL) {
+                if (fdctx->migration_failed) {
+                        error = 1;
+                }
+        }
+
+        return error;
+}
+
+#define FUSE_FD_GET_ACTIVE_FD(activefd, basefd)                 \
+        do {                                                    \
+                LOCK (&basefd->lock);                           \
+                {                                               \
+                        activefd = basefd_ctx->activefd ?       \
+                                basefd_ctx->activefd : basefd;  \
+                        if (activefd != basefd) {               \
+                                fd_ref (activefd);              \
+                        }                                       \
+                }                                               \
+                UNLOCK (&basefd->lock);                         \
+                                                                \
+                if (activefd == basefd) {                       \
+                        fd_ref (activefd);                      \
+                }                                               \
+        } while (0);
+
+
+static int
+fuse_resolve_fd (fuse_state_t *state)
+{
+        fuse_resolve_t *resolve            = NULL;
+	fd_t           *basefd             = NULL, *activefd = NULL;
+	xlator_t       *active_subvol      = NULL, *this = NULL;
+        int             ret                = 0;
+        char            fd_migration_error = 0;
+        fuse_fd_ctx_t  *basefd_ctx         = NULL;
+
+        resolve = state->resolve_now;
+
+        this = state->this;
+
+        basefd = resolve->fd;
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        if (basefd_ctx == NULL) {
+                gf_log (state->this->name, GF_LOG_WARNING,
+                        "fdctx is NULL for basefd (ptr:%p inode-gfid:%s), "
+                        "resolver erroring out with errno EINVAL",
+                        basefd, uuid_utoa (basefd->inode->gfid));
+                resolve->op_ret = -1;
+                resolve->op_errno = EINVAL;
+                goto resolve_continue;
+        }
+
+        FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
+
+        active_subvol = activefd->inode->table->xl;
+
+        fd_migration_error = fuse_migrate_fd_error (state->this, basefd);
+        if (fd_migration_error) {
+                resolve->op_ret = -1;
+                resolve->op_errno = EBADF;
+        } else if (state->active_subvol != active_subvol) {
+                ret = synctask_new (state->this->ctx->env, fuse_migrate_fd_task,
+                                    NULL, NULL, state);
+
+                fd_migration_error = fuse_migrate_fd_error (state->this,
+                                                            basefd);
+                fd_unref (activefd);
+
+                FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
+                active_subvol = activefd->inode->table->xl;
+
+                if ((ret == -1) || fd_migration_error
+                    || (state->active_subvol != active_subvol)) {
+                        if (ret == -1) {
+                                gf_log (state->this->name, GF_LOG_WARNING,
+                                        "starting sync-task to migrate "
+                                        "basefd (ptr:%p inode-gfid:%s) failed "
+                                        "(old-subvolume:%s-%d "
+                                        "new-subvolume:%s-%d)",
+                                        basefd,
+                                        uuid_utoa (basefd->inode->gfid),
+                                        active_subvol->name,
+                                        active_subvol->graph->id,
+                                        state->active_subvol->name,
+                                        state->active_subvol->graph->id);
+                        } else {
+                                gf_log (state->this->name, GF_LOG_WARNING,
+                                        "fd migration of basefd "
+                                        "(ptr:%p inode-gfid:%s) failed "
+                                        "(old-subvolume:%s-%d "
+                                        "new-subvolume:%s-%d)",
+                                        basefd,
+                                        uuid_utoa (basefd->inode->gfid),
+                                        active_subvol->name,
+                                        active_subvol->graph->id,
+                                        state->active_subvol->name,
+                                        state->active_subvol->graph->id);
+                        }
+
+                        resolve->op_ret = -1;
+                        resolve->op_errno = EBADF;
+                } else {
+                        gf_log (state->this->name, GF_LOG_DEBUG,
+                                "basefd (ptr:%p inode-gfid:%s) migrated "
+                                "successfully in resolver "
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                basefd, uuid_utoa (basefd->inode->gfid),
+                                active_subvol->name, active_subvol->graph->id,
+                                state->active_subvol->name,
+                                state->active_subvol->graph->id);
+                }
+        }
+
+        if ((resolve->op_ret == -1) && (resolve->op_errno == EBADF)) {
+                gf_log ("fuse-resolve", GF_LOG_WARNING,
+                        "migration of basefd (ptr:%p inode-gfid:%s) "
+                        "did not complete, failing fop with EBADF "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid),
+                        active_subvol->name, active_subvol->graph->id,
+                        state->active_subvol->name,
+                        state->active_subvol->graph->id);
+        }
+
+        if (activefd != basefd) {
+                state->fd = fd_ref (activefd);
+                fd_unref (basefd);
+        }
+
+	/* state->active_subvol = active_subvol; */
+
+resolve_continue:
+        if (activefd != NULL) {
+                fd_unref (activefd);
+        }
+
+        fuse_resolve_continue (state);
+
+        return 0;
+}
+
+
+int
+fuse_gfid_set (fuse_state_t *state)
+{
+        int   ret = 0;
+
+        if (uuid_is_null (state->gfid))
+                goto out;
+
+        if (!state->xdata)
+                state->xdata = dict_new ();
+
+        if (!state->xdata) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_static_bin (state->xdata, "gfid-req",
+                                   state->gfid, sizeof (state->gfid));
+out:
+        return ret;
+}
+
+
+int
+fuse_resolve_entry_init (fuse_state_t *state, fuse_resolve_t *resolve,
+			 ino_t par, char *name)
+{
+	inode_t       *parent = NULL;
+
+	parent = fuse_ino_to_inode (par, state->this);
+	uuid_copy (resolve->pargfid, parent->gfid);
+	resolve->parhint = parent;
+	resolve->bname = gf_strdup (name);
+
+	return 0;
+}
+
+
+int
+fuse_resolve_inode_init (fuse_state_t *state, fuse_resolve_t *resolve,
+			 ino_t ino)
+{
+	inode_t       *inode = NULL;
+
+	inode = fuse_ino_to_inode (ino, state->this);
+	uuid_copy (resolve->gfid, inode->gfid);
+	resolve->hint = inode;
+
+	return 0;
+}
+
+
+int
+fuse_resolve_fd_init (fuse_state_t *state, fuse_resolve_t *resolve,
+		      fd_t *fd)
+{
+	resolve->fd = fd_ref (fd);
+
+	return 0;
+}
+
+
+static int
+fuse_resolve (fuse_state_t *state)
+{
+        fuse_resolve_t   *resolve = NULL;
+
+        resolve = state->resolve_now;
+
+        if (resolve->fd) {
+
+                fuse_resolve_fd (state);
+
+        } else if (!uuid_is_null (resolve->pargfid)) {
+
+                fuse_resolve_parent (state);
+
+        } else if (!uuid_is_null (resolve->gfid)) {
+
+                fuse_resolve_inode (state);
+
+        } else {
+                fuse_resolve_all (state);
+        }
+
+        return 0;
+}
+
+
+static int
+fuse_resolve_done (fuse_state_t *state)
+{
+        fuse_resume_fn_t fn = NULL;
+
+        fn = state->resume_fn;
+
+	fn (state);
+
+        return 0;
+}
+
+
+/*
+ * This function is called multiple times, once per resolving one location/fd.
+ * state->resolve_now is used to decide which location/fd is to be resolved now
+ */
+static int
+fuse_resolve_all (fuse_state_t *state)
+{
+        if (state->resolve_now == NULL) {
+
+                state->resolve_now = &state->resolve;
+                state->loc_now     = &state->loc;
+
+                fuse_resolve (state);
+
+        } else if (state->resolve_now == &state->resolve) {
+
+                state->resolve_now = &state->resolve2;
+                state->loc_now     = &state->loc2;
+
+                fuse_resolve (state);
+
+        } else if (state->resolve_now == &state->resolve2) {
+
+                fuse_resolve_done (state);
+
+        } else {
+                gf_log ("fuse-resolve", GF_LOG_ERROR,
+                        "Invalid pointer for state->resolve_now");
+        }
+
+        return 0;
+}
+
+
+int
+fuse_resolve_continue (fuse_state_t *state)
+{
+        fuse_resolve_loc_touchup (state);
+
+        fuse_resolve_all (state);
+
+        return 0;
+}
+
+
+int
+fuse_resolve_and_resume (fuse_state_t *state, fuse_resume_fn_t fn)
+{
+        fuse_gfid_set (state);
+
+        state->resume_fn = fn;
+
+        fuse_resolve_all (state);
+
+        return 0;
+}
diff --git a/xlators/mount/fuse/utils/Makefile.am b/xlators/mount/fuse/utils/Makefile.am
index 1217c30da..c626e2769 100644
--- a/xlators/mount/fuse/utils/Makefile.am
+++ b/xlators/mount/fuse/utils/Makefile.am
@@ -1,4 +1,4 @@
-utildir = $(destdir)/sbin
+utildir = @mountutildir@
 
 if GF_DARWIN_HOST_OS
 util_SCRIPTS = mount_glusterfs
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index 481fd265f..a192d6059 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -1,23 +1,21 @@
 #!/bin/sh
-# (C) 2006, 2007, 2008 Z RESEARCH Inc. <http://www.zresearch.com>
-# 
+# (C) 2006, 2007, 2008 Gluster Inc. <http://www.gluster.com>
+#
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as
 # published by the Free Software Foundation; either version 2 of
 # the License, or (at your option) any later version.
-#   
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 # GNU General Public License for more details.
-#   
+#
 # You should have received a copy of the GNU General Public
 # License along with this program; if not, write to the Free
 # Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 # Boston, MA 02110-1301 USA
 
-
-
 _init ()
 {
     # log level definitions
@@ -25,127 +23,444 @@ _init ()
     LOG_CRITICAL=CRITICAL;
     LOG_ERROR=ERROR;
     LOG_WARNING=WARNING;
+    LOG_INFO=INFO
     LOG_DEBUG=DEBUG;
+    LOG_TRACE=TRACE;
 
-    # set default log level to ERROR
-    log_level=$LOG_WARNING;
-}
+    HOST_NAME_MAX=64;
 
-start_glusterfs ()
-{
     prefix="@prefix@";
     exec_prefix=@exec_prefix@;
     cmd_line=$(echo "@sbindir@/glusterfs");
-    
+
+    case `uname -s` in
+    NetBSD)
+        getinode="stat -f %i"
+        getdev="stat -f %d"
+        lgetinode="${getinode} -L"
+        lgetdev="${getdev} -L"
+
+        mounttab=/proc/mounts
+        ;;
+    Linux)
+        getinode="stat -c %i $i"
+        getdev="stat -c %d $d"
+        lgetinode="${getinode} -L"
+        lgetdev="${getdev} -L"
+
+        mounttab=/etc/mtab
+        ;;
+    esac
+
+    UPDATEDBCONF=/etc/updatedb.conf
+}
+
+parse_backup_volfile_servers ()
+{
+    local server_list=$1
+    local servers=""
+    local new_servers=""
+
+    servers=$(echo ${server_list} | sed 's/\:/ /g')
+    for server in ${servers}; do
+        length=$(echo $server | wc -c)
+        if [ ${length} -gt ${HOST_NAME_MAX} ]; then
+            echo "Hostname:${server} provided is too long.. skipping"
+            continue
+        fi
+        new_servers=$(echo "$new_servers $server")
+    done
+    echo ${new_servers}
+}
+
+start_glusterfs ()
+{
+    # lets the comparsion be case insensitive for all strings
+
     if [ -n "$log_level_str" ]; then
-	case "$log_level_str" in
-	    "ERROR")
-		log_level=$LOG_ERROR;
-		;;
-	    "DEBUG")
-		log_level=$LOG_DEBUG;
-		;;
-	    "CRITICAL")
-		log_level=$LOG_CRITICAL;
-		;;
-	    "WARNING")
-		log_level=$LOG_WARNING;
-		;;
-	    "NONE")
-		log_level=$LOG_NONE;
-		;;
-	    *)
-		echo "invalid log level $log_level_str, using ERROR";
-		log_level=$LOG_ERROR;
-		;;
-	esac
-    fi
-    cmd_line=$(echo "$cmd_line --log-level=$log_level");
-    
+        case "$( echo $log_level_str | tr '[a-z]' '[A-Z]')" in
+            "ERROR")
+                log_level=$LOG_ERROR;
+                ;;
+            "INFO")
+                log_level=$LOG_INFO
+                ;;
+            "DEBUG")
+                log_level=$LOG_DEBUG;
+                ;;
+            "CRITICAL")
+                log_level=$LOG_CRITICAL;
+                ;;
+            "WARNING")
+                log_level=$LOG_WARNING;
+                ;;
+            "TRACE")
+                log_level=$LOG_TRACE;
+                ;;
+            "NONE")
+                log_level=$LOG_NONE;
+                ;;
+            *)
+                echo "invalid log level $log_level_str, using INFO";
+                log_level=$LOG_INFO;
+                ;;
+        esac
+    fi
+
+#options without values start here
+    if [ -n "$read_only" ]; then
+        cmd_line=$(echo "$cmd_line --read-only");
+    fi
+
+    if [ -n "$acl" ]; then
+        cmd_line=$(echo "$cmd_line --acl");
+    fi
+
+    if [ -n "$selinux" ]; then
+        cmd_line=$(echo "$cmd_line --selinux");
+    fi
+
+    if [ -n "$enable_ino32" ]; then
+        cmd_line=$(echo "$cmd_line --enable-ino32");
+    fi
+
+    if [ -n "$worm" ]; then
+        cmd_line=$(echo "$cmd_line --worm");
+    fi
+
+    if [ -n "$fopen_keep_cache" ]; then
+        cmd_line=$(echo "$cmd_line --fopen-keep-cache");
+    fi
+
+    if [ -n "$volfile_check" ]; then
+        cmd_line=$(echo "$cmd_line --volfile-check");
+    fi
+
+    if [ -n "$mem_accounting" ]; then
+        cmd_line=$(echo "$cmd_line --mem-accounting");
+    fi
+
+    if [ -n "$aux_gfid_mount" ]; then
+        cmd_line=$(echo "$cmd_line --aux-gfid-mount");
+    fi
+
+#options with values start here
+    if [ -n "$log_level" ]; then
+        cmd_line=$(echo "$cmd_line --log-level=$log_level");
+    fi
+
     if [ -n "$log_file" ]; then
-	cmd_line=$(echo "$cmd_line --log-file=$log_file");
+        cmd_line=$(echo "$cmd_line --log-file=$log_file");
     fi
 
     if [ -n "$direct_io_mode" ]; then
-	cmd_line=$(echo "$cmd_line --direct-io-mode=$direct_io_mode");
+        cmd_line=$(echo "$cmd_line --direct-io-mode=$direct_io_mode");
     fi
-    
-    if [ -z "$volfile_loc" ]; then
-	if [ -n "$transport" ]; then 
-	    cmd_line=$(echo "$cmd_line \
---volfile-server=$server_ip \
---volfile-server-port=$server_port \
---volfile-server-transport=$transport");
-	else
-	    cmd_line=$(echo "$cmd_line \
---volfile-server=$server_ip \
---volfile-server-port=$server_port");
-	fi
-    else
-	cmd_line=$(echo "$cmd_line --volfile=$volfile_loc");
+
+    if [ -n "$use_readdirp" ]; then
+        cmd_line=$(echo "$cmd_line --use-readdirp=$use_readdirp");
     fi
 
     if [ -n "$volume_name" ]; then
         cmd_line=$(echo "$cmd_line --volume-name=$volume_name");
     fi
-    
-    if [ -n "$volume_id" ]; then
-        cmd_line=$(echo "$cmd_line --volfile-id=$volume_id");
+
+    if [ -n "$attribute_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --attribute-timeout=$attribute_timeout");
+    fi
+
+    if [ -n "$entry_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --entry-timeout=$entry_timeout");
+    fi
+
+    if [ -n "$negative_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --negative-timeout=$negative_timeout");
+    fi
+
+    if [ -n "$gid_timeout" ]; then
+        cmd_line=$(echo "$cmd_line --gid-timeout=$gid_timeout");
+    fi
+
+    if [ -n "$bg_qlen" ]; then
+        cmd_line=$(echo "$cmd_line --background-qlen=$bg_qlen");
+    fi
+
+    if [ -n "$cong_threshold" ]; then
+        cmd_line=$(echo "$cmd_line --congestion-threshold=$cong_threshold");
+    fi
+
+    if [ -n "$fuse_mountopts" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-mountopts=$fuse_mountopts");
+    fi
+
+    if [ -n "$xlator_option" ]; then
+        xlator_option=$(echo $xlator_option | sed s/"xlator-option="/"--xlator-option "/g)
+        cmd_line=$(echo "$cmd_line $xlator_option");
+    fi
+
+    # for rdma volume, we have to fetch volfile with '.rdma' added
+    # to volume name, so that it fetches the right client vol file
+    volume_id_rdma="";
+
+    if [ -z "$volfile_loc" ]; then
+        if  [ -n "$server_ip" ]; then
+
+            cmd_line=$(echo "$cmd_line --volfile-server=$server_ip");
+
+            if [ -n "$backup_volfile_servers" ]; then
+                servers=$(parse_backup_volfile_servers ${backup_volfile_servers})
+                for i in $(echo ${servers}); do
+                    cmd_line=$(echo "$cmd_line --volfile-server=$i");
+                done
+            fi
+
+            if [ -n "$server_port" ]; then
+                cmd_line=$(echo "$cmd_line --volfile-server-port=$server_port");
+            fi
+
+            if [ -n "$transport" ]; then
+                cmd_line=$(echo "$cmd_line --volfile-server-transport=$transport");
+                if [ "$transport" = "rdma" ]; then
+                    volume_id_rdma=".rdma";
+                fi
+            fi
+
+            if [ -n "$volume_id" ]; then
+                if [ -n "$volume_id_rdma" ]; then
+                    volume_id="$volume_id$volume_id_rdma";
+                fi
+                cmd_line=$(echo "$cmd_line --volfile-id=$volume_id");
+            fi
+        fi
+    else
+        cmd_line=$(echo "$cmd_line --volfile=$volfile_loc");
+    fi
+
+    if [ -n "$fuse_mountopts" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-mountopts=$fuse_mountopts");
     fi
 
     cmd_line=$(echo "$cmd_line $mount_point");
-    exec $cmd_line;
+    err=0;
+    $cmd_line;
+
+
+    inode=$( ${getinode} $mount_point 2>/dev/null);
+
+    # this is required if the stat returns error
+    if [ -z "$inode" ]; then
+        inode="0";
+    fi
+
+    if [ $inode -ne 1 ]; then
+        err=1;
+    fi
+
+    if [ $err -eq "1" ]; then
+            echo "Mount failed. Please check the log file for more details."
+            umount $mount_point > /dev/null 2>&1;
+            exit 1;
+    fi
 }
 
+usage ()
+{
 
-main ()
+echo "Usage:  mount.glusterfs <volumeserver>:<volumeid/volumeport> -o <options> <mountpoint>
+Options:
+man 8 mount.glusterfs
+
+To display the version number of the mount helper:
+mount.glusterfs --version"
+
+}
+
+# check for recursive mounts. i.e, mounting over an existing brick
+check_recursive_mount ()
 {
-    options=$(echo "$@" | sed -n 's/.*\-o[ ]*\([^ ]*\).*/\1/p');
-    new_log_level=$(echo "$options" | sed -n 's/.*log-level=\([^,]*\).*/\1/p');
-    
-    [ -n "$new_log_level" ] && {
-	log_level_str="$new_log_level";
-    }
-    log_file=$(echo "$options" | sed -n 's/.*log-file=\([^,]*\).*/\1/p');
+    if [ $1 = "/" ]; then
+        echo Cannot mount over root;
+        exit 2;
+    fi
+    # GFID check first
+    # remove trailing / from mount point
+    mnt_dir=${1%/};
+
+    export PATH;
+    # check whether getfattr exists
+    which getfattr > /dev/null 2>&1;
+    if [ $? -ne 0 ]; then
+        return;
+    fi
+
+    getfattr -n trusted.gfid $mnt_dir 2>/dev/null | grep -iq "trusted.gfid=";
+    if [ $? -eq 0 ]; then
+        echo "ERROR: $mnt_dir is in use as a brick of a gluster volume";
+        exit 2;
+    fi
+
+    # check if the mount point is a brick's parent directory
+    GLUSTERD_WORKDIR="/var/lib/glusterd";
+
+    ls -L "$GLUSTERD_WORKDIR"/vols/*/bricks/* > /dev/null 2>&1;
+    if [ $? -ne 0 ]; then
+        return;
+    fi
 
-    transport=$(echo "$options" | sed -n 's/.*transport=\([^,]*\).*/\1/p');
+    brick_path=`grep ^path "$GLUSTERD_WORKDIR"/vols/*/bricks/* | cut -d "=" -f 2`;
+    root_inode=`${lgetinode} /`;
+    root_dev=`${lgetdev} /`;
+    mnt_inode=`${lgetinode} $mnt_dir`;
+    mnt_dev=`${lgetdev} $mnt_dir`;
+    for brick in "$brick_path";
+    do
+        # evaluate brick path to see if this is local, if non-local, skip iteration
+        ls $brick > /dev/null 2>&1;
+        if [ $? -ne 0 ]; then
+            continue;
+        fi
+        getfattr -n trusted.gfid "$brick" 2>/dev/null | grep -iq "trusted.gfid=";
+        if [ $? -ne 0 ]; then
+            continue;
+        else
+            # brick is local
+            while [ 1 ];
+            do
+                tmp_brick="$brick";
+                brick="$brick"/..;
+                brick_dev=`${lgetdev} $brick`;
+                brick_inode=`${lgetinode} $brick`;
+                if [ "$mnt_inode" -eq "$brick_inode" -a "$mnt_dev" -eq "$brick_dev" ]; then
+                    echo ERROR: $mnt_dir is a parent of the brick $tmp_brick;
+                    exit 2;
+                fi
+                [ "$root_inode" -ne "$brick_inode" -o "$root_dev" -ne "$brick_dev" ] || break;
+            done;
+        fi
+    done;
+}
 
-    direct_io_mode=$(echo "$options" | sed -n 's/.*direct-io-mode=\([^,]*\).*/\1/p');
+main ()
+{
+    helper=$(echo "$@" | sed -n 's/.*\--[ ]*\([^ ]*\).*/\1/p');
+    in_opt="no"
+    pos_args=0
+    for opt in "$@"; do
+        if [ "$in_opt" = "yes" ]; then
+            for pair in $(echo "$opt" | tr "," " "); do
+                # Handle options without values.
+                case "$pair" in
+                    "ro")               read_only=1 ;;
+                    "acl")              acl=1 ;;
+                    "selinux")          selinux=1 ;;
+                    "worm")             worm=1 ;;
+                    "fopen-keep-cache") fopen_keep_cache=1 ;;
+                    "enable-ino32")     enable_ino32=1 ;;
+                    "mem-accounting")   mem_accounting=1;;
+                    "aux-gfid-mount")
+                        if [ `uname -s` = "Linux" ]; then
+                            aux_gfid_mount=1
+                        fi
+                        ;;
+                    # "mount -t glusterfs" sends this, but it's useless.
+                    "rw")       ;;
+                    # these ones are interpreted during system initialization
+                    "noauto")   ;;
+                    "_netdev")  ;;
+                    *)
+                        key=$(echo "$pair" | cut -f1 -d'=');
+                        value=$(echo "$pair" | cut -f2- -d'=');
 
-    volume_name=$(echo "$options" | sed -n 's/.*volume-name=\([^,]*\).*/\1/p');
+                        # Handle options with values.
+                        case "$key" in
+                            "log-level")        log_level_str=$value ;;
+                            "log-file")         log_file=$value ;;
+                            "transport")        transport=$value ;;
+                            "direct-io-mode")   direct_io_mode=$value ;;
+                            "volume-name")      volume_name=$value ;;
+                            "volume-id")        volume_id=$value ;;
+                            "volfile-check")    volfile_check=$value ;;
+                            "server-port")      server_port=$value ;;
+                            "attribute-timeout")
+                                attribute_timeout=$value ;;
+                            "entry-timeout")    entry_timeout=$value ;;
+                            "negative-timeout")    negative_timeout=$value ;;
+                            "gid-timeout")   gid_timeout=$value ;;
+                            "background-qlen") bg_qlen=$value ;;
+                            "backup-volfile-servers") backup_volfile_servers=$value ;;
+                            "congestion-threshold")  cong_threshold=$value ;;
+                            "xlator-option")  xlator_option=$xlator_option" "$pair ;;
+                            "fuse-mountopts")  fuse_mountopts=$value ;;
+                            "use-readdirp") use_readdirp=$value ;;
+                            *)
+                                # Passthru
+                                [ -z "$fuse_mountopts" ] || fuse_mountopts="$fuse_mountopts,"
+                                fuse_mountopts="$fuse_mountopts$pair"
+                                ;;
+                        esac
+                esac
+            done
+            in_opt="no"
+        elif [ "$opt" = "-o" ]; then
+            in_opt="yes"
+        else
+            case $pos_args in
+                0)      volfile_loc=$opt ;;
+                1)      mount_point=$opt ;;
+                *)      echo "extra arguments at end (ignored)" ;;
+            esac
+            pos_args=$((pos_args+1))
+        fi
+    done
+    if [ $in_opt = "yes" -o $pos_args -lt 2 ]; then
+        usage
+        exit 1
+    fi
 
-    volume_id=$(echo "$options" | sed -n 's/.*volume-id=\([^,]*\).*/\1/p');
-    
-    volfile_loc="$1";
-    
     [ -r "$volfile_loc" ] || {
-	server_ip=$(echo "$volfile_loc" | sed -n 's/\([^\:]*\).*/\1/p');
-	server_port=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
-	[ -n "$server_port" ] || {
-	    server_port="6996";
-	}
+        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        test_str=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
+        [ -n "$test_str" ] && {
+            volume_id="$test_str";
+        }
+        volfile_loc="";
+    }
 
-	volfile_loc="";
+    #
+    [ -n "$helper" ] && {
+        cmd_line=$(echo "$cmd_line --$helper");
+        exec $cmd_line;
+        exit 0;
+    }
+
+    # No need to do a ! -d test, it is taken care while initializing the
+    # variable mount_point
+    [ -z "$mount_point" -o ! -d "$mount_point" ] && {
+        echo "ERROR: Mount point does not exist."
+        usage;
+        exit 0;
     }
-    new_fs_options=$(echo "$options" | sed -e 's/[,]*log-file=[^,]*//' \
-	                                   -e 's/[,]*log-level=[^,]*//' \
-	                                   -e 's/[,]*volume-name=[^,]*//' \
-	                                   -e 's/[,]*direct-io-mode=[^,]*//' \
-	                                   -e 's/[,]*transport=[^,]*//' \
-	                                   -e 's/[,]*volume-id=[^,]*//');
-    # following line is product of love towards sed
-    # $2=$(echo "$@" | sed -n 's/[^ ]* \([^ ]*\).*/\1/p');
-    
-    mount_point="$2";
 
     # Simple check to avoid multiple identical mounts
-    if grep -q "glusterfs $mount_point fuse" /etc/mtab; then
-        echo "$0: according to mtab, GlusterFS is already mounted on $mount_point"
-        exit 1
+    if grep -q "[[:space:]+]${mount_point}[[:space:]+]fuse" $mounttab; then
+        echo -n "$0: according to mtab, GlusterFS is already mounted on "
+        echo "$mount_point"
+        exit 0;
     fi
-    
-    fs_options=$(echo "$fs_options,$new_fs_options");
-    
+
+    check_recursive_mount "$mount_point";
+
+    # Append fuse.glusterfs to PRUNEFS variable in updatedb.conf(5). updatedb(8)
+    # should not index files under GlusterFS, indexing will slow down GlusteFS
+    # if the filesystem is several TB in size.
+    test -f $UPDATEDBCONF && {
+        if ! grep -q 'glusterfs' $UPDATEDBCONF; then
+            sed 's/\(PRUNEFS.*\)"/\1 fuse.glusterfs"/' $UPDATEDBCONF \
+                > ${UPDATEDBCONF}.bak
+            mv -f ${UPDATEDBCONF}.bak $UPDATEDBCONF
+        fi
+    }
+
     start_glusterfs;
 }
 
diff --git a/xlators/mount/fuse/utils/mount_glusterfs.in b/xlators/mount/fuse/utils/mount_glusterfs.in
index 1376a8897..b12b4e04e 100755
--- a/xlators/mount/fuse/utils/mount_glusterfs.in
+++ b/xlators/mount/fuse/utils/mount_glusterfs.in
@@ -1,5 +1,5 @@
 #!/bin/sh
-# (C) 2008 Z RESEARCH Inc. <http://www.zresearch.com>
+# (C) 2008 Gluster Inc. <http://www.gluster.com>
 # 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as
@@ -25,10 +25,11 @@ _init ()
     LOG_CRITICAL=CRITICAL;
     LOG_ERROR=ERROR;
     LOG_WARNING=WARNING;
+    LOG_INFO=INFO;
     LOG_DEBUG=DEBUG;
 
     # set default log level to ERROR
-    log_level=$LOG_WARNING;
+    log_level=$LOG_INFO;
 }
 
 start_glusterfs ()
@@ -42,6 +43,9 @@ start_glusterfs ()
 	    "ERROR")
 		log_level=$LOG_ERROR;
 		;;
+            "INFO")
+                log_level=$LOG_INFO;
+                ;;
 	    "DEBUG")
 		log_level=$LOG_DEBUG;
 		;;
@@ -55,8 +59,8 @@ start_glusterfs ()
 		log_level=$LOG_NONE;
 		;;
 	    *)
-		echo "invalid log level $log_level_str, using ERROR";
-		log_level=$LOG_WARNING;
+		echo "invalid log level $log_level_str, using INFO";
+		log_level=$LOG_INFO;
 		;;
 	esac
     fi
@@ -66,20 +70,22 @@ start_glusterfs ()
 	cmd_line=$(echo "$cmd_line --log-file=$log_file");
     fi
 
+    if [ -n "$volfile_check" ]; then
+	cmd_line=$(echo "$cmd_line --volfile-check");
+    fi
+
     if [ -n "$direct_io_mode" ]; then
-	cmd_line=$(echo "$cmd_line --direct-io-mode=$direct_io_mode");
+	cmd_line=$(echo "$cmd_line --disable-direct-io-mode");
     fi
     
     if [ -z "$volfile_loc" ]; then
 	if [ -n "$transport" ]; then 
 	    cmd_line=$(echo "$cmd_line \
 --volfile-server=$server_ip \
---volfile-server-port=$server_port \
 --volfile-server-transport=$transport");
 	else
 	    cmd_line=$(echo "$cmd_line \
---volfile-server=$server_ip \
---volfile-server-port=$server_port");
+--volfile-server=$server_ip");
 	fi
     else
 	cmd_line=$(echo "$cmd_line --volfile=$volfile_loc");
@@ -107,6 +113,7 @@ main ()
     direct_io_mode=""
     volume_name=""
     new_fs_options=""
+    volfile_check=""
 
     while getopts o: opt; do
 	case "$opt" in
@@ -128,6 +135,10 @@ main ()
 		    direct_io_mode=$(echo "$options" | sed -n 's/.*direct-io-mode=\([^,]*\).*/\1/p');
 		}
 		
+		[ -z $volfile_check ] && {
+		    volfile_check=$(echo "$options" | sed -n 's/.*volfile-check=\([^,]*\).*/\1/p');
+		}
+		
 		[ -z $volume_name ] && {
 		    volume_name=$(echo "$options" | sed -n 's/.*volume-name=\([^,]*\).*/\1/p');
 		}
@@ -139,6 +150,7 @@ main ()
 		this_option=$(echo "$options" | sed -e 's/[,]*log-file=[^,]*//' \
 		    -e 's/[,]*log-level=[^,]*//' \
 		    -e 's/[,]*volume-name=[^,]*//' \
+		    -e 's/[,]*volfile-check=[^,]*//' \
 		    -e 's/[,]*direct-io-mode=[^,]*//' \
 		    -e 's/[,]*transport=[^,]*//' \
 		    -e 's/[,]*volume-id=[^,]*//');
@@ -153,19 +165,15 @@ main ()
 
     # TODO: use getopt. This is very much darwin specific
     volfile_loc="$1";
-    while [ "$volfile_loc" == "-o" ] ; do
+    while [ "$volfile_loc" = "-o" ] ; do
 	shift ;
 	shift ;
 	volfile_loc="$1";
     done
     
     [ -r "$volfile_loc" ] || {
-	server_ip=$(echo "$volfile_loc" | sed -n 's/\([^\:]*\).*/\1/p');
-	server_port=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
-	[ -n "$server_port" ] || {
-	    server_port="6996";
-	}
-
+        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        volume_id=$(echo "$volfile_loc" | sed -n 's/[a-zA-Z0-9:.\-]*:\(.*\)/\1/p');
 	volfile_loc="";
     }
     # following line is product of love towards sed
diff --git a/xlators/nfs/Makefile.am b/xlators/nfs/Makefile.am
new file mode 100644
index 000000000..8771032f6
--- /dev/null
+++ b/xlators/nfs/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = server
+
+CLEANFILES =
diff --git a/xlators/nfs/server/Makefile.am b/xlators/nfs/server/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/nfs/server/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/nfs/server/src/Makefile.am b/xlators/nfs/server/src/Makefile.am
new file mode 100644
index 000000000..62fbf6535
--- /dev/null
+++ b/xlators/nfs/server/src/Makefile.am
@@ -0,0 +1,24 @@
+xlator_LTLIBRARIES = server.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/nfs
+nfsrpclibdir = $(top_srcdir)/rpc/rpc-lib/src
+server_la_LDFLAGS = -module -avoid-version
+server_la_SOURCES = nfs.c nfs-common.c nfs-fops.c nfs-inodes.c \
+	nfs-generics.c mount3.c nfs3-fh.c nfs3.c nfs3-helpers.c nlm4.c \
+	nlmcbk_svc.c mount3udp_svc.c acl3.c
+server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = nfs.h nfs-common.h nfs-fops.h nfs-inodes.h nfs-generics.h \
+	mount3.h nfs3-fh.h nfs3.h nfs3-helpers.h nfs-mem-types.h nlm4.h \
+	acl3.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" \
+	-I$(top_srcdir)/libglusterfs/src \
+	-I$(nfsrpclibdir) -I$(CONTRIBDIR)/rbtree \
+	-I$(top_srcdir)/rpc/xdr/src/
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+AM_LDFLAGS = -L$(xlatordir)
+
+CLEANFILES =
diff --git a/xlators/nfs/server/src/acl3.c b/xlators/nfs/server/src/acl3.c
new file mode 100644
index 000000000..08b099b4e
--- /dev/null
+++ b/xlators/nfs/server/src/acl3.c
@@ -0,0 +1,708 @@
+/*
+ * Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "rpcsvc.h"
+#include "dict.h"
+#include "xlator.h"
+#include "nfs.h"
+#include "mem-pool.h"
+#include "logging.h"
+#include "nfs-fops.h"
+#include "inode.h"
+#include "nfs3.h"
+#include "nfs-mem-types.h"
+#include "nfs3-helpers.h"
+#include "nfs3-fh.h"
+#include "nfs-generics.h"
+#include "acl3.h"
+
+
+typedef ssize_t (*acl3_serializer) (struct iovec outmsg, void *args);
+
+extern void nfs3_call_state_wipe (nfs3_call_state_t *cs);
+
+extern nfs3_call_state_t *
+nfs3_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v);
+
+extern int
+nfs3_fh_validate (struct nfs3_fh *fh);
+
+extern fattr3
+nfs3_stat_to_fattr3 (struct iatt *buf);
+
+#define acl3_validate_nfs3_state(request, state, status, label, retval) \
+        do      {                                                       \
+                state = rpcsvc_request_program_private (request);       \
+                if (!state) {                                           \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "NFSv3 state "    \
+                                "missing from RPC request");            \
+                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
+                        status = NFS3ERR_SERVERFAULT;                   \
+                        goto label;                                     \
+                }                                                       \
+        } while (0);                                                    \
+
+#define acl3_validate_gluster_fh(handle, status, errlabel)              \
+        do {                                                            \
+                if (!nfs3_fh_validate (handle)) {                       \
+                        status = NFS3ERR_SERVERFAULT;                   \
+                        goto errlabel;                                  \
+                }                                                       \
+        } while (0)                                                     \
+
+
+extern xlator_t *
+nfs3_fh_to_xlator (struct nfs3_state *nfs3, struct nfs3_fh *fh);
+
+#define acl3_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
+        do {                                                            \
+                char exportid[256], gfid[256];                          \
+                rpc_transport_t *trans = NULL;                          \
+                volume = nfs3_fh_to_xlator ((nfs3state), handle);      \
+                if (!volume) {                                          \
+                        uuid_unparse (handle->exportid, exportid);       \
+                        uuid_unparse (handle->gfid, gfid);               \
+                        trans = rpcsvc_request_transport (req);         \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Failed to map "  \
+                                "FH to vol: client=%s, exportid=%s, gfid=%s",\
+                                trans->peerinfo.identifier, exportid,   \
+                                gfid);                                  \
+                        gf_log (GF_ACL, GF_LOG_ERROR,                   \
+                                "Stale nfs client %s must be trying to "\
+                                "connect to a deleted volume, please "  \
+                                "unmount it.", trans->peerinfo.identifier);\
+                        status = NFS3ERR_STALE;                         \
+                        goto label;                                     \
+                } else {                                                \
+                        gf_log (GF_ACL, GF_LOG_TRACE, "FH to Volume: %s"\
+                                ,volume->name);                         \
+                        rpcsvc_request_set_private (req, volume);       \
+                }                                                       \
+        } while (0);                                                    \
+
+#define acl3_volume_started_check(nfs3state, vlm, rtval, erlbl)         \
+        do {                                                            \
+              if ((!nfs_subvolume_started (nfs_state (nfs3state->nfsx), vlm))){\
+                      gf_log (GF_ACL, GF_LOG_ERROR, "Volume is disabled: %s",\
+                              vlm->name);                               \
+                      rtval = RPCSVC_ACTOR_IGNORE;                      \
+                      goto erlbl;                                       \
+              }                                                         \
+        } while (0)                                                     \
+
+#define acl3_check_fh_resolve_status(cst, nfstat, erlabl)               \
+        do {                                                            \
+                xlator_t *xlatorp = NULL;                               \
+                char buf[256], gfid[256];                               \
+                rpc_transport_t *trans = NULL;                          \
+                if ((cst)->resolve_ret < 0) {                           \
+                        trans = rpcsvc_request_transport (cst->req);    \
+                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
+                                                     &cst->resolvefh);  \
+                        uuid_unparse (cst->resolvefh.gfid, gfid);       \
+                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
+                                  trans->peerinfo.identifier,           \
+                                  xlatorp ? xlatorp->name : "ERR",      \
+                                  gfid);                                \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Unable to resolve FH"\
+                                ": %s", buf);                           \
+                        nfstat = nfs3_errno_to_nfsstat3 (cst->resolve_errno);\
+                        goto erlabl;                                    \
+                }                                                       \
+        } while (0)                                                     \
+
+#define acl3_handle_call_state_init(nfs3state, calls, rq, v, opstat, errlabel)\
+        do {                                                            \
+                calls = nfs3_call_state_init ((nfs3state), (rq), v); \
+                if (!calls) {                                           \
+                        gf_log (GF_ACL, GF_LOG_ERROR, "Failed to "      \
+                                "init call state");                     \
+                        opstat = NFS3ERR_SERVERFAULT;                   \
+                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
+                        goto errlabel;                                  \
+                }                                                       \
+        } while (0)                                                     \
+
+
+int
+acl3svc_submit_reply (rpcsvc_request_t *req, void *arg, acl3_serializer sfunc)
+{
+        struct iovec            outmsg  = {0, };
+        struct iobuf            *iob    = NULL;
+        struct nfs3_state       *nfs3   = NULL;
+        int                     ret     = -1;
+        ssize_t                 msglen  = 0;
+        struct iobref           *iobref = NULL;
+
+        if (!req)
+                return -1;
+
+        nfs3 = (struct nfs3_state *)rpcsvc_request_program_private (req);
+        if (!nfs3) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "mount state not found");
+                goto ret;
+        }
+
+        /* First, get the io buffer into which the reply in arg will
+         * be serialized.
+         */
+        iob = iobuf_get (nfs3->iobpool);
+        if (!iob) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to get iobuf");
+                goto ret;
+        }
+
+        iobuf_to_iovec (iob, &outmsg);
+        /* Use the given serializer to translate the give C structure in arg
+         * to XDR format which will be written into the buffer in outmsg.
+         */
+        msglen = sfunc (outmsg, arg);
+        if (msglen < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to encode message");
+                goto ret;
+        }
+        outmsg.iov_len = msglen;
+
+        iobref = iobref_new ();
+        if (iobref == NULL) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to get iobref");
+                goto ret;
+        }
+
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
+
+        /* Then, submit the message for transmission. */
+        ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Reply submission failed");
+                goto ret;
+        }
+
+        ret = 0;
+ret:
+        if (iob)
+                iobuf_unref (iob);
+        if (iobref)
+                iobref_unref (iobref);
+
+        return ret;
+}
+
+
+int
+acl3svc_null (rpcsvc_request_t *req)
+{
+        struct iovec    dummyvec = {0, };
+
+        if (!req) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Got NULL request!");
+                return 0;
+        }
+        rpcsvc_submit_generic (req, &dummyvec, 1,  NULL, 0, NULL);
+        return 0;
+}
+
+int
+acl3_getacl_reply (nfs3_call_state_t *cs, getaclreply *reply)
+{
+        acl3svc_submit_reply (cs->req, (void *)reply,
+                              (acl3_serializer)xdr_serialize_getaclreply);
+        return 0;
+}
+
+
+int
+acl3_getacl_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict,
+                 dict_t *xdata)
+{
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs3_call_state_t              *cs = NULL;
+        data_t                         *data = NULL;
+        int                            *p = NULL;
+        int                             i = 0;
+        getaclreply                     *getaclreply = NULL;
+
+        if (!frame->local) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Invalid argument,"
+                       " frame->local NULL");
+                return  EINVAL;
+        }
+        cs = frame->local;
+        getaclreply = &cs->args.getaclreply;
+        if (op_ret == -1) {
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
+                goto err;
+        }
+
+        getaclreply->aclentry.aclentry_val = cs->aclentry;
+        getaclreply->daclentry.daclentry_val = cs->daclentry;
+
+        /* FIXME: use posix_acl_from_xattr() */
+        data = dict_get (dict, POSIX_ACL_ACCESS_XATTR);
+        if (data && (p = data_to_bin (data))) {
+                /* POSIX_ACL_VERSION */
+                p++;
+                while ((char *)p < (data->data + data->len)) {
+                        getaclreply->aclentry.aclentry_val[i].type = *(*(short **)&p)++;
+                        getaclreply->aclentry.aclentry_val[i].perm = *(*(short **)&p)++;
+                        getaclreply->aclentry.aclentry_val[i].uid = *(*(int **)&p)++;
+                        i++;
+                }
+                getaclreply->aclcount = getaclreply->aclentry.aclentry_len = i;
+        }
+        i = 0;
+
+        data = dict_get (dict, POSIX_ACL_DEFAULT_XATTR);
+        if (data && (p = data_to_bin (data))) {
+                /* POSIX_ACL_VERSION */
+                p++;
+                while ((char *)p < (data->data + data->len)) {
+                        getaclreply->daclentry.daclentry_val[i].type = *(*(short **)&p)++;
+                        getaclreply->daclentry.daclentry_val[i].perm = *(*(short **)&p)++;
+                        getaclreply->daclentry.daclentry_val[i].uid = *(*(int **)&p)++;
+                        i++;
+                }
+                getaclreply->daclcount = getaclreply->daclentry.daclentry_len = i;
+        }
+
+        acl3_getacl_reply (cs, getaclreply);
+        nfs3_call_state_wipe (cs);
+        return 0;
+
+err:
+        if (getaclreply)
+                getaclreply->status = stat;
+        acl3_getacl_reply (cs, getaclreply);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+int
+acl3_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs3_call_state_t              *cs = NULL;
+        getaclreply                     *getaclreply = NULL;
+        int                             ret = -1;
+        nfs_user_t                      nfu = {0, };
+
+        if (!frame->local) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Invalid argument,"
+                       " frame->local NULL");
+                return EINVAL;
+        }
+
+        cs = frame->local;
+        getaclreply = &cs->args.getaclreply;
+
+        if (op_ret == -1) {
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
+                goto err;
+        }
+
+        getaclreply->attr_follows = 1;
+        getaclreply->attr = nfs3_stat_to_fattr3 (buf);
+        getaclreply->mask = 0xf;
+        nfs_request_user_init (&nfu, cs->req);
+        ret = nfs_getxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, NULL, NULL,
+                            acl3_getacl_cbk, cs);
+        if (ret == -1) {
+                stat = nfs3_cbk_errno_status (op_ret, op_errno);
+                goto err;
+        }
+        return 0;
+err:
+        getaclreply->status = stat;
+        acl3_getacl_reply (cs, getaclreply);
+        nfs3_call_state_wipe (cs);
+        return 0;
+}
+
+
+int
+acl3_getacl_resume (void *carg)
+{
+        int                             ret = -1;
+        nfs3_call_state_t               *cs = NULL;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs_user_t                      nfu = {0, };
+
+        if (!carg)
+                return ret;
+
+        cs = (nfs3_call_state_t *)carg;
+        acl3_check_fh_resolve_status (cs, stat, acl3err);
+        nfs_request_user_init (&nfu, cs->req);
+
+        ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                        acl3_stat_cbk, cs);
+        stat = -ret;
+acl3err:
+        if (ret < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to open_and_resume");
+                cs->args.getaclreply.status = nfs3_errno_to_nfsstat3 (stat);
+                acl3_getacl_reply (cs, &cs->args.getaclreply);
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+
+int
+acl3svc_getacl (rpcsvc_request_t *req)
+{
+        xlator_t                        *vol = NULL;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t               *cs = NULL;
+        int                             ret = RPCSVC_ACTOR_ERROR;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        struct nfs3_fh                  fh, *fhp = NULL;
+        getaclargs                      getaclargs;
+
+        if (!req)
+                return ret;
+
+        acl3_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        memset (&getaclargs, 0, sizeof (getaclargs));
+        getaclargs.fh.n_bytes = (char *)&fh;
+        if (xdr_to_getaclargs(req->msg[0], &getaclargs) <= 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+        fhp = &fh;
+        acl3_validate_gluster_fh (&fh, stat, acl3err);
+        acl3_map_fh_to_volume (nfs->nfs3state, fhp, req,
+                               vol, stat, acl3err);
+        acl3_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     vol, stat, rpcerr);
+
+        cs->vol = vol;
+        acl3_volume_started_check (nfs3, vol, ret, acl3err);
+
+        ret = nfs3_fh_resolve_and_resume (cs, fhp,
+                                          NULL, acl3_getacl_resume);
+
+acl3err:
+        if (ret < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to resolve and resume");
+                if (cs) {
+                        cs->args.getaclreply.status = stat;
+                        acl3_getacl_reply (cs, &cs->args.getaclreply);
+                        nfs3_call_state_wipe (cs);
+                }
+                return 0;
+        }
+
+rpcerr:
+        return ret;
+}
+
+int
+acl3_setacl_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno,
+                 dict_t *xdata)
+{
+        nfs3_call_state_t               *cs = NULL;
+        cs = frame->local;
+        if (op_ret < 0) {
+                nfsstat3 status = nfs3_cbk_errno_status (op_ret, op_errno);
+                cs->args.setaclreply.status = status;
+        }
+
+        acl3svc_submit_reply (cs->req, (void *)&cs->args.setaclreply,
+                              (acl3_serializer)xdr_serialize_setaclreply);
+        return 0;
+}
+
+int
+acl3_setacl_resume (void *carg)
+{
+        int                             ret = -1;
+        nfs3_call_state_t               *cs = NULL;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        nfs_user_t                      nfu = {0, };
+        dict_t                          *xattr = NULL;
+
+        if (!carg)
+                return ret;
+        cs = (nfs3_call_state_t *)carg;
+        acl3_check_fh_resolve_status (cs, stat, acl3err);
+        nfs_request_user_init (&nfu, cs->req);
+        xattr = dict_new();
+        if (cs->aclcount)
+        ret = dict_set_static_bin (xattr, POSIX_ACL_ACCESS_XATTR, cs->aclxattr,
+                                   cs->aclcount * 8 + 4);
+        if (cs->daclcount)
+        ret = dict_set_static_bin (xattr, POSIX_ACL_DEFAULT_XATTR,
+                                   cs->daclxattr, cs->daclcount * 8 + 4);
+
+        ret = nfs_setxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, xattr,
+                            0, NULL, acl3_setacl_cbk, cs);
+        dict_unref (xattr);
+
+acl3err:
+        if (ret < 0) {
+                stat = -ret;
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to open_and_resume");
+                cs->args.setaclreply.status = nfs3_errno_to_nfsstat3 (stat);
+                acl3svc_submit_reply (cs->req, (void *)&cs->args.setaclreply,
+                                      (acl3_serializer)xdr_serialize_setaclreply);
+                nfs3_call_state_wipe (cs);
+        }
+
+        return ret;
+}
+
+
+int
+acl3svc_setacl (rpcsvc_request_t *req)
+{
+        xlator_t                        *vol = NULL;
+        struct nfs_state               *nfs = NULL;
+        nfs3_state_t                   *nfs3 = NULL;
+        nfs3_call_state_t               *cs = NULL;
+        int                             ret = RPCSVC_ACTOR_ERROR;
+        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
+        struct nfs3_fh                  fh;
+        struct nfs3_fh                 *fhp = NULL;
+        setaclargs                      setaclargs;
+        aclentry                        *aclentry = NULL;
+        struct aclentry                 *daclentry = NULL;
+        int                             i = 0;
+        struct posix_acl_xattr_header   *bufheader = NULL;
+        struct posix_acl_xattr_entry    *bufentry  = NULL;
+
+        if (!req)
+                return ret;
+        aclentry =  GF_CALLOC (NFS_ACL_MAX_ENTRIES, sizeof(*aclentry),
+                               gf_nfs_mt_arr);
+        if (!aclentry) {
+                goto rpcerr;
+        }
+        daclentry = GF_CALLOC (NFS_ACL_MAX_ENTRIES, sizeof(*daclentry),
+                               gf_nfs_mt_arr);
+        if (!daclentry) {
+                goto rpcerr;
+        }
+
+        acl3_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
+        nfs = nfs_state (nfs3->nfsx);
+        memset (&setaclargs, 0, sizeof (setaclargs));
+        memset (&fh, 0, sizeof (fh));
+        setaclargs.fh.n_bytes = (char *)&fh;
+        setaclargs.aclentry.aclentry_val = aclentry;
+        setaclargs.daclentry.daclentry_val = daclentry;
+        if (xdr_to_setaclargs(req->msg[0], &setaclargs) <= 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Error decoding args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+        fhp = &fh;
+        acl3_validate_gluster_fh (fhp, stat, acl3err);
+        acl3_map_fh_to_volume (nfs->nfs3state, fhp, req,
+                               vol, stat, acl3err);
+        acl3_handle_call_state_init (nfs->nfs3state, cs, req,
+                                     vol, stat, rpcerr);
+
+        cs->vol = vol;
+        acl3_volume_started_check (nfs3, vol, ret, rpcerr);
+
+        cs->aclcount = setaclargs.aclcount;
+        cs->daclcount = setaclargs.daclcount;
+
+        if ((cs->aclcount > NFS_ACL_MAX_ENTRIES) ||
+                        (cs->daclcount > NFS_ACL_MAX_ENTRIES))
+                goto acl3err;
+        /* FIXME: use posix_acl_to_xattr() */
+        /* Populate xattr buffer for user ACL */
+        bufheader = (struct posix_acl_xattr_header *)(cs->aclxattr);
+        bufheader->version = htole32(POSIX_ACL_VERSION);
+        bufentry  = bufheader->entries;
+        for (i = 0; i < cs->aclcount; i++) {
+                int uaceuid;
+                const struct aclentry *uace = &aclentry[i];
+                switch (uace->type) {
+                    case POSIX_ACL_USER:
+                    case POSIX_ACL_GROUP:
+                        uaceuid = uace->uid;
+                        break;
+                    default:
+                        uaceuid = POSIX_ACL_UNDEFINED_ID;
+                        break;
+                }
+                bufentry->tag  = htole16(uace->type);
+                bufentry->perm = htole16(uace->perm);
+                bufentry->id   = htole32(uaceuid);
+
+                bufentry++;
+        }
+
+        /* Populate xattr buffer for Default ACL */
+        bufheader = (struct posix_acl_xattr_header *)(cs->aclxattr);
+        bufheader->version = htole32(POSIX_ACL_VERSION);
+        bufentry  = bufheader->entries;
+        for (i = 0; i < cs->daclcount; i++) {
+                int daceuid;
+                int dacetype;
+                const struct aclentry *dace = &daclentry[i];
+                /*
+                 * For "default ACL", NFSv3 handles the 'type' differently
+                 * i.e. by logical OR'ing 'type' with NFS_ACL_DEFAULT.
+                 * Which the backend File system does not understand and
+                 * that needs to be masked OFF.
+                 */
+                dacetype = (dace->type & ~(NFS_ACL_DEFAULT));
+                switch (dacetype) {
+                    case POSIX_ACL_USER:
+                    case POSIX_ACL_GROUP:
+                        daceuid = dace->uid;
+                        break;
+                    default:
+                        daceuid = POSIX_ACL_UNDEFINED_ID;
+                        break;
+                }
+                bufentry->tag  = htole16(dacetype);
+                bufentry->perm = htole16(dace->perm);
+                bufentry->id   = htole32(daceuid);
+
+                bufentry++;
+        }
+
+
+        ret = nfs3_fh_resolve_and_resume (cs, fhp,
+                                          NULL, acl3_setacl_resume);
+
+acl3err:
+        if (ret < 0) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "unable to resolve and resume");
+                cs->args.setaclreply.status = stat;
+                acl3svc_submit_reply (cs->req, (void *)&cs->args.setaclreply,
+                                      (acl3_serializer)xdr_serialize_setaclreply);
+                nfs3_call_state_wipe (cs);
+                GF_FREE(aclentry);
+                GF_FREE(daclentry);
+                return 0;
+        }
+
+rpcerr:
+        if (ret < 0)
+                nfs3_call_state_wipe (cs);
+        if (aclentry)
+                GF_FREE (aclentry);
+        if (daclentry)
+                GF_FREE (daclentry);
+        return ret;
+}
+
+
+
+rpcsvc_actor_t  acl3svc_actors[ACL3_PROC_COUNT] = {
+        {"NULL",       ACL3_NULL,      acl3svc_null,   NULL,   0},
+        {"GETACL",     ACL3_GETACL,    acl3svc_getacl, NULL,   0},
+        {"SETACL",     ACL3_SETACL,    acl3svc_setacl, NULL,   0},
+};
+
+rpcsvc_program_t        acl3prog = {
+        .progname       = "ACL3",
+        .prognum        = ACL_PROGRAM,
+        .progver        = ACLV3_VERSION,
+        .progport       = GF_NFS3_PORT,
+        .actors         = acl3svc_actors,
+        .numactors      = ACL3_PROC_COUNT,
+        .min_auth       = AUTH_NULL,
+};
+
+rpcsvc_program_t *
+acl3svc_init(xlator_t *nfsx)
+{
+        struct nfs3_state *ns = NULL;
+        struct nfs_state *nfs = NULL;
+        dict_t *options = NULL;
+        int ret = -1;
+        char *portstr = NULL;
+        static gf_boolean_t acl3_inited = _gf_false;
+
+        /* Already inited */
+        if (acl3_inited)
+                return &acl3prog;
+
+        nfs = (struct nfs_state*)nfsx->private;
+
+        ns = nfs->nfs3state;
+        if (!ns) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "ACL3 init failed");
+                goto err;
+        }
+        acl3prog.private = ns;
+
+        options = dict_new ();
+
+        ret = gf_asprintf (&portstr, "%d", GF_ACL3_PORT);
+        if (ret == -1)
+                goto err;
+
+        ret = dict_set_dynstr (options, "transport.socket.listen-port",
+                               portstr);
+        if (ret == -1)
+                goto err;
+        ret = dict_set_str (options, "transport-type", "socket");
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        if (nfs->allow_insecure) {
+                ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
+                if (ret == -1) {
+                        gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                        goto err;
+                }
+                ret = dict_set_str (options, "rpc-auth.ports.insecure", "on");
+                if (ret == -1) {
+                        gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                        goto err;
+                }
+        }
+
+        ret = dict_set_str (options, "transport.address-family", "inet");
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
+                goto err;
+        }
+
+        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, "ACL");
+        if (ret == -1) {
+                gf_log (GF_ACL, GF_LOG_ERROR, "Unable to create listeners");
+                dict_unref (options);
+                goto err;
+        }
+
+        acl3_inited = _gf_true;
+        return &acl3prog;
+err:
+        return NULL;
+}
diff --git a/xlators/nfs/server/src/acl3.h b/xlators/nfs/server/src/acl3.h
new file mode 100644
index 000000000..e0e61281a
--- /dev/null
+++ b/xlators/nfs/server/src/acl3.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ */
+
+#ifndef _ACL3_H
+#define _ACL3_H
+
+#include "glusterfs-acl.h"
+
+#define GF_ACL3_PORT            38469
+#define GF_ACL                  GF_NFS"-ACL"
+
+/*
+ * NFSv3, identifies the default ACL by NFS_ACL_DEFAULT. Gluster
+ * NFS needs to mask it OFF before sending it upto POSIX layer
+ * or File system layer.
+ */
+#define NFS_ACL_DEFAULT             0x1000
+
+#define NFS_ACL_MAX_ENTRIES         1024
+
+rpcsvc_program_t *
+acl3svc_init(xlator_t *nfsx);
+
+#endif
diff --git a/xlators/nfs/server/src/mount3.c b/xlators/nfs/server/src/mount3.c
new file mode 100644
index 000000000..b0824bf10
--- /dev/null
+++ b/xlators/nfs/server/src/mount3.c
@@ -0,0 +1,2759 @@
+/*
+  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "rpcsvc.h"
+#include "dict.h"
+#include "xlator.h"
+#include "mount3.h"
+#include "xdr-nfs3.h"
+#include "msg-nfs3.h"
+#include "iobuf.h"
+#include "nfs-common.h"
+#include "nfs3-fh.h"
+#include "nfs-fops.h"
+#include "nfs-inodes.h"
+#include "nfs-generics.h"
+#include "locking.h"
+#include "iatt.h"
+#include "nfs-mem-types.h"
+#include "nfs.h"
+#include "common-utils.h"
+#include "store.h"
+
+#include <errno.h>
+#include <sys/socket.h>
+#include <sys/uio.h>
+
+
+#define IPv4_ADDR_SIZE  32
+
+/* Macro to typecast the parameter to struct sockaddr_in
+ */
+#define SA(addr) ((struct sockaddr_in*)(addr))
+
+/* Macro will mask the ip address with netmask.
+ */
+#define MASKED_IP(ipv4addr, netmask)                    \
+                (ntohl(SA(ipv4addr)->sin_addr.s_addr) & (netmask))
+
+/* Macro will compare two IP address after applying the mask
+ */
+#define COMPARE_IPv4_ADDRS(ip1, ip2, netmask)           \
+                ((MASKED_IP(ip1, netmask)) == (MASKED_IP(ip2, netmask)))
+
+/* This macro will assist in freeing up entire link list
+ * of host_auth_spec structure.
+ */
+#define FREE_HOSTSPEC(exp) do {                                 \
+                struct host_auth_spec *host= exp->hostspec;     \
+                while (NULL != host){                           \
+                        struct host_auth_spec* temp = host;     \
+                        host = host->next;                      \
+                        if (NULL != temp->host_addr) {          \
+                                GF_FREE (temp->host_addr);      \
+                        }                                       \
+                        GF_FREE (temp);                         \
+                }                                               \
+                exp->hostspec = NULL;                           \
+        } while (0)
+
+typedef ssize_t (*mnt3_serializer) (struct iovec outmsg, void *args);
+
+extern void *
+mount3udp_thread (void *argv);
+
+static inline void
+mnt3_export_free (struct mnt3_export *exp)
+{
+        if (!exp)
+                return;
+
+        if (exp->exptype == MNT3_EXPTYPE_DIR)
+                FREE_HOSTSPEC (exp);
+        GF_FREE (exp->expname);
+        GF_FREE (exp);
+}
+
+/* Generic reply function for MOUNTv3 specific replies. */
+int
+mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
+{
+        struct iovec            outmsg  = {0, };
+        struct iobuf            *iob    = NULL;
+        struct mount3_state     *ms     = NULL;
+        int                     ret     = -1;
+        ssize_t                 msglen  = 0;
+        struct iobref           *iobref = NULL;
+
+        if (!req)
+                return -1;
+
+        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
+        if (!ms) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "mount state not found");
+                goto ret;
+        }
+
+        /* First, get the io buffer into which the reply in arg will
+         * be serialized.
+         */
+        /* TODO: use 'xdrproc_t' instead of 'sfunc' to get the xdr-size */
+        iob = iobuf_get (ms->iobpool);
+        if (!iob) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get iobuf");
+                goto ret;
+        }
+
+        iobuf_to_iovec (iob, &outmsg);
+        /* Use the given serializer to translate the give C structure in arg
+         * to XDR format which will be written into the buffer in outmsg.
+         */
+        msglen = sfunc (outmsg, arg);
+        if (msglen < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to encode message");
+                goto ret;
+        }
+        outmsg.iov_len = msglen;
+
+        iobref = iobref_new ();
+        if (iobref == NULL) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get iobref");
+                goto ret;
+        }
+
+        ret = iobref_add (iobref, iob);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to add iob to iobref");
+                goto ret;
+        }
+
+        /* Then, submit the message for transmission. */
+        ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
+        if (ret == -1) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Reply submission failed");
+                goto ret;
+        }
+
+        ret = 0;
+ret:
+        if (NULL != iob)
+                iobuf_unref (iob);
+        if (NULL != iobref)
+                iobref_unref (iobref);
+
+        return ret;
+}
+
+
+/* Generic error reply function, just pass the err status
+ * and it will do the rest, including transmission.
+ */
+int
+mnt3svc_mnt_error_reply (rpcsvc_request_t *req, int mntstat)
+{
+        mountres3       res;
+
+        if (!req)
+                return -1;
+
+        res.fhs_status = mntstat;
+        mnt3svc_submit_reply (req, (void *)&res,
+                              (mnt3_serializer)xdr_serialize_mountres3);
+
+        return 0;
+}
+
+
+mountstat3
+mnt3svc_errno_to_mnterr (int32_t errnum)
+{
+        mountstat3      stat;
+
+        switch (errnum) {
+
+                case 0:
+                        stat = MNT3_OK;
+                        break;
+                case ENOENT:
+                        stat = MNT3ERR_NOENT;
+                        break;
+                case EPERM:
+                        stat = MNT3ERR_PERM;
+                        break;
+                case EIO:
+                        stat = MNT3ERR_IO;
+                        break;
+                case EACCES:
+                        stat = MNT3ERR_ACCES;
+                        break;
+                case ENOTDIR:
+                        stat = MNT3ERR_NOTDIR;
+                        break;
+                case EINVAL:
+                        stat = MNT3ERR_INVAL;
+                        break;
+                case ENOSYS:
+                        stat = MNT3ERR_NOTSUPP;
+                        break;
+                case ENOMEM:
+                        stat = MNT3ERR_SERVERFAULT;
+                        break;
+                default:
+                        stat = MNT3ERR_SERVERFAULT;
+                        break;
+        }
+
+        return stat;
+}
+
+
+mountres3
+mnt3svc_set_mountres3 (mountstat3 stat, struct nfs3_fh *fh, int *authflavor,
+                       u_int aflen)
+{
+        mountres3       res = {0, };
+        uint32_t        fhlen = 0;
+
+        res.fhs_status = stat;
+        fhlen = nfs3_fh_compute_size (fh);
+        res.mountres3_u.mountinfo.fhandle.fhandle3_len = fhlen;
+        res.mountres3_u.mountinfo.fhandle.fhandle3_val = (char *)fh;
+        res.mountres3_u.mountinfo.auth_flavors.auth_flavors_val = authflavor;
+        res.mountres3_u.mountinfo.auth_flavors.auth_flavors_len = aflen;
+
+        return res;
+}
+
+/* Read the rmtab from the store_handle and append (or not) the entries to the
+ * mountlist.
+ *
+ * Requires the store_handle to be locked.
+ */
+static int
+__mount_read_rmtab (gf_store_handle_t *sh, struct list_head *mountlist,
+                    gf_boolean_t append)
+{
+        int                     ret = 0;
+        unsigned int            idx = 0;
+        struct mountentry       *me = NULL, *tmp = NULL;
+                                /* me->hostname is a char[MNTPATHLEN] */
+        char                    key[MNTPATHLEN + 11];
+
+        GF_ASSERT (sh && mountlist);
+
+        if (!gf_store_locked_local (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Not reading unlocked %s",
+                        sh->path);
+                return -1;
+        }
+
+        if (!append) {
+                list_for_each_entry_safe (me, tmp, mountlist, mlist) {
+                        list_del (&me->mlist);
+                        GF_FREE (me);
+                }
+                me = NULL;
+        }
+
+        for (;;) {
+                char *value = NULL;
+
+                if (me && append) {
+                        /* do not add duplicates */
+                        list_for_each_entry (tmp, mountlist, mlist) {
+                                if (!strcmp(tmp->hostname, me->hostname) &&
+                                    !strcmp(tmp->exname, me->exname)) {
+                                        GF_FREE (me);
+                                        goto dont_add;
+                                }
+                        }
+                        list_add_tail (&me->mlist, mountlist);
+                } else if (me) {
+                        list_add_tail (&me->mlist, mountlist);
+                }
+
+dont_add:
+                me = GF_CALLOC (1, sizeof (*me), gf_nfs_mt_mountentry);
+                if (!me) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory");
+                        ret = -1;
+                        goto out;
+                }
+
+                INIT_LIST_HEAD (&me->mlist);
+
+                snprintf (key, 9 + MNTPATHLEN, "hostname-%d", idx);
+                ret = gf_store_retrieve_value (sh, key, &value);
+                if (ret)
+                        break;
+                strncpy (me->hostname, value, MNTPATHLEN);
+                GF_FREE (value);
+
+                snprintf (key, 11 + MNTPATHLEN, "mountpoint-%d", idx);
+                ret = gf_store_retrieve_value (sh, key, &value);
+                if (ret)
+                        break;
+                strncpy (me->exname, value, MNTPATHLEN);
+                GF_FREE (value);
+
+                idx++;
+                gf_log (GF_MNT, GF_LOG_TRACE, "Read entries %s:%s", me->hostname, me->exname);
+        }
+        gf_log (GF_MNT, GF_LOG_DEBUG, "Read %d entries from '%s'", idx, sh->path);
+        GF_FREE (me);
+out:
+        return ret;
+}
+
+/* Overwrite the contents of the rwtab with te in-memory client list.
+ * Fail gracefully if the stora_handle is not locked.
+ */
+static void
+__mount_rewrite_rmtab(struct mount3_state *ms, gf_store_handle_t *sh)
+{
+        struct mountentry       *me = NULL;
+        char                    key[16];
+        int                     fd, ret;
+        unsigned int            idx = 0;
+
+        if (!gf_store_locked_local (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Not modifying unlocked %s",
+                        sh->path);
+                return;
+        }
+
+        fd = gf_store_mkstemp (sh);
+        if (fd == -1) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to open %s", sh->path);
+                return;
+        }
+
+        list_for_each_entry (me, &ms->mountlist, mlist) {
+                snprintf (key, 16, "hostname-%d", idx);
+                ret = gf_store_save_value (fd, key, me->hostname);
+                if (ret)
+                        goto fail;
+
+                snprintf (key, 16, "mountpoint-%d", idx);
+                ret = gf_store_save_value (fd, key, me->exname);
+                if (ret)
+                        goto fail;
+
+                idx++;
+        }
+
+        gf_log (GF_MNT, GF_LOG_DEBUG, "Updated rmtab with %d entries", idx);
+
+        close (fd);
+        if (gf_store_rename_tmppath (sh))
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to overwrite rwtab %s",
+                        sh->path);
+
+        return;
+
+fail:
+        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to update %s", sh->path);
+        close (fd);
+        gf_store_unlink_tmppath (sh);
+}
+
+/* Read the rmtab into a clean ms->mountlist.
+ */
+static void
+mount_read_rmtab (struct mount3_state *ms)
+{
+        gf_store_handle_t       *sh = NULL;
+        struct nfs_state        *nfs = NULL;
+        int                     ret;
+
+        nfs = (struct nfs_state *)ms->nfsx->private;
+
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                return;
+        }
+
+        if (gf_store_lock (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to lock '%s'",
+                        nfs->rmtab);
+                goto out;
+        }
+
+        __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
+        gf_store_unlock (sh);
+
+out:
+        gf_store_handle_destroy (sh);
+}
+
+/* Write the ms->mountlist to the rmtab.
+ *
+ * The rmtab could be empty, or it can exists and have been updated by a
+ * different storage server without our knowing.
+ *
+ * 1. takes the store_handle lock on the current rmtab
+ *    - blocks if an other storage server rewrites the rmtab at the same time
+ * 2. [if new_rmtab] takes the store_handle lock on the new rmtab
+ * 3. reads/merges the entries from the current rmtab
+ * 4. [if new_rmtab] reads/merges the entries from the new rmtab
+ * 5. [if new_rmtab] writes the new rmtab
+ * 6. [if not new_rmtab] writes the current rmtab
+ * 7  [if new_rmtab] replaces nfs->rmtab to point to the new location
+ * 8. [if new_rmtab] releases the store_handle lock of the new rmtab
+ * 9. releases the store_handle lock of the old rmtab
+ */
+void
+mount_rewrite_rmtab (struct mount3_state *ms, char *new_rmtab)
+{
+        gf_store_handle_t       *sh = NULL, *nsh = NULL;
+        struct nfs_state        *nfs = NULL;
+        int                     ret;
+        char                    *rmtab = NULL;
+
+        nfs = (struct nfs_state *)ms->nfsx->private;
+
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                return;
+        }
+
+        if (gf_store_lock (sh)) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Not rewriting '%s'",
+                        nfs->rmtab);
+                goto free_sh;
+        }
+
+        if (new_rmtab) {
+                ret = gf_store_handle_new (new_rmtab, &nsh);
+                if (ret) {
+                        gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                                new_rmtab);
+                        goto unlock_sh;
+                }
+
+                if (gf_store_lock (nsh)) {
+                        gf_log (GF_MNT, GF_LOG_WARNING, "Not rewriting '%s'",
+                                new_rmtab);
+                        goto free_nsh;
+                }
+        }
+
+        /* always read the currently used rmtab */
+        __mount_read_rmtab (sh, &ms->mountlist, _gf_true);
+
+        if (new_rmtab) {
+                /* read the new rmtab and write changes to the new location */
+                __mount_read_rmtab (nsh, &ms->mountlist, _gf_true);
+                __mount_rewrite_rmtab (ms, nsh);
+
+                /* replace the nfs->rmtab reference to the new rmtab */
+                rmtab = gf_strdup(new_rmtab);
+                if (rmtab == NULL) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory, keeping "
+                                "%s as rmtab", nfs->rmtab);
+                } else {
+                        GF_FREE (nfs->rmtab);
+                        nfs->rmtab = new_rmtab;
+                }
+
+                gf_store_unlock (nsh);
+        } else {
+                /* rewrite the current (unchanged location) rmtab */
+                __mount_rewrite_rmtab (ms, sh);
+        }
+
+free_nsh:
+        if (new_rmtab)
+                gf_store_handle_destroy (nsh);
+unlock_sh:
+        gf_store_unlock (sh);
+free_sh:
+        gf_store_handle_destroy (sh);
+}
+
+/* Add a new NFS-client to the ms->mountlist and update the rmtab if we can.
+ *
+ * A NFS-client will only be removed from the ms->mountlist in case the
+ * NFS-client sends a unmount request. It is possible that a NFS-client
+ * crashed/rebooted had network loss or something else prevented the NFS-client
+ * to unmount cleanly. In this case, a duplicate entry would be added to the
+ * ms->mountlist, which is wrong and we should prevent.
+ *
+ * It is fully acceptible that the ms->mountlist is not 100% correct, this is a
+ * common issue for all(?) NFS-servers.
+ */
+int
+mnt3svc_update_mountlist (struct mount3_state *ms, rpcsvc_request_t *req,
+                          char  *expname)
+{
+        struct mountentry       *me = NULL;
+        struct mountentry       *cur = NULL;
+        int                     ret = -1;
+        char                    *colon = NULL;
+        struct nfs_state        *nfs = NULL;
+        gf_store_handle_t       *sh = NULL;
+
+        if ((!ms) || (!req) || (!expname))
+                return -1;
+
+        me = (struct mountentry *)GF_CALLOC (1, sizeof (*me),
+                                             gf_nfs_mt_mountentry);
+        if (!me)
+                return -1;
+
+        nfs = (struct nfs_state *)ms->nfsx->private;
+
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                goto free_err;
+        }
+
+        strncpy (me->exname, expname, MNTPATHLEN);
+
+        INIT_LIST_HEAD (&me->mlist);
+        /* Must get the IP or hostname of the client so we
+         * can map it into the mount entry.
+         */
+        ret = rpcsvc_transport_peername (req->trans, me->hostname, MNTPATHLEN);
+        if (ret == -1)
+                goto free_err2;
+
+        colon = strrchr (me->hostname, ':');
+        if (colon) {
+                *colon = '\0';
+        }
+        LOCK (&ms->mountlock);
+        {
+                /* in case locking fails, we just don't write the rmtab */
+                if (gf_store_lock (sh)) {
+                        gf_log (GF_MNT, GF_LOG_WARNING, "Failed to lock '%s'"
+                                ", changes will not be written", nfs->rmtab);
+                } else {
+                        __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
+                }
+
+                /* do not add duplicates */
+                list_for_each_entry (cur, &ms->mountlist, mlist) {
+                        if (!strcmp(cur->hostname, me->hostname) &&
+                            !strcmp(cur->exname, me->exname)) {
+                                GF_FREE (me);
+                                goto dont_add;
+                        }
+                }
+                list_add_tail (&me->mlist, &ms->mountlist);
+
+                /* only write the rmtab in case it was locked */
+                if (gf_store_locked_local (sh))
+                        __mount_rewrite_rmtab (ms, sh);
+        }
+dont_add:
+        if (gf_store_locked_local (sh))
+                gf_store_unlock (sh);
+
+        UNLOCK (&ms->mountlock);
+
+free_err2:
+        gf_store_handle_destroy (sh);
+
+free_err:
+        if (ret == -1)
+                GF_FREE (me);
+
+        return ret;
+}
+
+
+int
+__mnt3_get_volume_id (struct mount3_state *ms, xlator_t *mntxl,
+                      uuid_t volumeid)
+{
+        int                     ret = -1;
+        struct mnt3_export      *exp = NULL;
+
+        if ((!ms) || (!mntxl))
+                return ret;
+
+        LOCK (&ms->mountlock);
+        list_for_each_entry (exp, &ms->exportlist, explist) {
+                if (exp->vol == mntxl) {
+                        uuid_copy (volumeid, exp->volumeid);
+                        ret = 0;
+                        goto out;
+                }
+        }
+
+out:
+        UNLOCK (&ms->mountlock);
+        return ret;
+}
+
+
+int32_t
+mnt3svc_lookup_mount_cbk (call_frame_t *frame, void  *cookie,
+                          xlator_t *this, int32_t op_ret, int32_t op_errno,
+                          inode_t *inode, struct iatt *buf, dict_t *xattr,
+                          struct iatt *postparent)
+{
+        mountres3               res = {0, };
+        rpcsvc_request_t        *req = NULL;
+        struct nfs3_fh          fh = {{0}, };
+        struct mount3_state     *ms = NULL;
+        mountstat3              status = 0;
+        int                     autharr[10];
+        int                     autharrlen = 0;
+        rpcsvc_t                *svc = NULL;
+        xlator_t                *mntxl = NULL;
+        uuid_t                  volumeid = {0, };
+        char                    fhstr[1024], *path = NULL;
+
+        req = (rpcsvc_request_t *)frame->local;
+
+        if (!req)
+                return -1;
+
+        mntxl = (xlator_t *)cookie;
+        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
+        if (!ms) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "mount state not found");
+                op_ret = -1;
+                op_errno = EINVAL;
+        }
+
+        if (op_ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR, "error=%s", strerror (op_errno));
+                status = mnt3svc_errno_to_mnterr (op_errno);
+        }
+        if (status != MNT3_OK)
+                goto xmit_res;
+
+        path = GF_CALLOC (PATH_MAX, sizeof (char), gf_nfs_mt_char);
+        if (!path) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory");
+                goto xmit_res;
+        }
+
+        snprintf (path, PATH_MAX, "/%s", mntxl->name);
+        mnt3svc_update_mountlist (ms, req, path);
+        GF_FREE (path);
+        if (gf_nfs_dvm_off (nfs_state (ms->nfsx))) {
+                fh = nfs3_fh_build_indexed_root_fh (ms->nfsx->children, mntxl);
+                goto xmit_res;
+        }
+
+        __mnt3_get_volume_id (ms, mntxl, volumeid);
+        fh = nfs3_fh_build_uuid_root_fh (volumeid);
+
+xmit_res:
+        nfs3_fh_to_str (&fh, fhstr, sizeof (fhstr));
+        gf_log (GF_MNT, GF_LOG_DEBUG, "MNT reply: fh %s, status: %d", fhstr,
+                status);
+        if (op_ret == 0) {
+                svc = rpcsvc_request_service (req);
+                autharrlen = rpcsvc_auth_array (svc, mntxl->name, autharr,
+                                                10);
+        }
+
+        res = mnt3svc_set_mountres3 (status, &fh, autharr, autharrlen);
+        mnt3svc_submit_reply (req, (void *)&res,
+                              (mnt3_serializer)xdr_serialize_mountres3);
+
+        return 0;
+}
+
+
+int
+mnt3_match_dirpath_export (char *expname, char *dirpath)
+{
+        int     ret = 0;
+        size_t  dlen;
+
+        if ((!expname) || (!dirpath))
+                return 0;
+
+        /* Some clients send a dirpath for mount that includes the slash at the
+         * end. String compare for searching the export will fail because our
+         * exports list does not include that slash. Remove the slash to
+         * compare.
+         */
+        dlen = strlen (dirpath);
+        if (dlen && dirpath [dlen - 1] == '/')
+                dirpath [dlen - 1] = '\0';
+
+        if (dirpath[0] != '/')
+                expname++;
+
+        if (strcmp (expname, dirpath) == 0)
+                ret = 1;
+
+        return ret;
+}
+
+
+int
+mnt3svc_mount_inode (rpcsvc_request_t *req, struct mount3_state *ms,
+                     xlator_t * xl, inode_t *exportinode)
+{
+        int             ret = -EFAULT;
+        nfs_user_t      nfu = {0, };
+        loc_t           exportloc = {0, };
+
+        if ((!req) || (!xl) || (!ms) || (!exportinode))
+                return ret;
+
+        ret = nfs_inode_loc_fill (exportinode, &exportloc, NFS_RESOLVE_EXIST);
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Loc fill failed for export inode"
+                        ": gfid %s, volume: %s",
+                        uuid_utoa (exportinode->gfid), xl->name);
+                goto err;
+        }
+
+        /* To service the mount request, all we need to do
+         * is to send a lookup fop that returns the stat
+         * for the root of the child volume. This is
+         * used to build the root fh sent to the client.
+         */
+        nfs_request_user_init (&nfu, req);
+        ret = nfs_lookup (ms->nfsx, xl, &nfu, &exportloc,
+                          mnt3svc_lookup_mount_cbk, (void *)req);
+
+        nfs_loc_wipe (&exportloc);
+err:
+        return ret;
+}
+
+
+/* For a volume mount request, we just have to create loc on the root inode,
+ * and send a lookup. In the lookup callback the mount reply is send along with
+ * the file handle.
+ */
+int
+mnt3svc_volume_mount (rpcsvc_request_t *req, struct mount3_state *ms,
+                      struct mnt3_export *exp)
+{
+        inode_t         *exportinode = NULL;
+        int             ret = -EFAULT;
+        uuid_t          rootgfid = {0, };
+
+        if ((!req) || (!exp) || (!ms))
+                return ret;
+
+        rootgfid[15] = 1;
+        exportinode = inode_find (exp->vol->itable, rootgfid);
+        if (!exportinode) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get root inode");
+                ret = -ENOENT;
+                goto err;
+        }
+
+        ret = mnt3svc_mount_inode (req, ms, exp->vol, exportinode);
+        inode_unref (exportinode);
+
+err:
+        return ret;
+}
+
+
+/* The catch with directory exports is that the first component of the export
+ * name will be the name of the volume.
+ * Any lookup that needs to be performed to build the directory's file handle
+ * needs to start from the directory path from the root of the volume. For that
+ * we need to strip out the volume name first.
+ */
+char *
+__volume_subdir (char *dirpath, char **volname)
+{
+        char    *subdir = NULL;
+        int     volname_len = 0;
+
+        if (!dirpath)
+                return NULL;
+
+        if (dirpath[0] == '/')
+                dirpath++;
+
+        subdir = index (dirpath, (int)'/');
+        if (!subdir)
+                goto out;
+
+        if (!volname)
+                goto out;
+
+        if (!*volname)
+                goto out;
+
+        /* subdir points to the first / after the volume name while dirpath
+         * points to the first char of the volume name.
+         */
+        volname_len = subdir - dirpath;
+        strncpy (*volname, dirpath, volname_len);
+        *(*volname + volname_len) = '\0';
+out:
+        return subdir;
+}
+
+
+void
+mnt3_resolve_state_wipe (mnt3_resolve_t *mres)
+{
+        if (!mres)
+                return;
+
+        nfs_loc_wipe (&mres->resolveloc);
+        GF_FREE (mres);
+
+}
+
+
+/* Sets up the component argument to contain the next component in the path and
+ * sets up path as an absolute path starting from the next component.
+ */
+static char *
+setup_next_component (char *path, size_t plen, char *component, size_t clen)
+{
+        char    *comp = NULL;
+        char    *nextcomp = NULL;
+
+        if ((!path) || (!component))
+                return NULL;
+
+        strncpy (component, path, clen);
+        comp = index (component, (int)'/');
+        if (!comp)
+                goto err;
+
+        comp++;
+        nextcomp = index (comp, (int)'/');
+        if (nextcomp) {
+                strncpy (path, nextcomp, plen);
+                *nextcomp = '\0';
+        } else
+                path[0] = '\0';
+
+err:
+        return comp;
+}
+
+int32_t
+mnt3_resolve_subdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, dict_t *xattr,
+                         struct iatt *postparent);
+
+/* There are multiple components in the directory export path and each one
+ * needs to be looked up one after the other.
+ */
+int
+__mnt3_resolve_export_subdir_comp (mnt3_resolve_t *mres)
+{
+        char            dupsubdir[MNTPATHLEN];
+        char            *nextcomp = NULL;
+        int             ret = -EFAULT;
+        nfs_user_t      nfu = {0, };
+        uuid_t          gfid = {0, };
+
+        if (!mres)
+                return ret;
+
+        nextcomp = setup_next_component (mres->remainingdir,
+                                         sizeof (mres->remainingdir),
+                                         dupsubdir, sizeof (dupsubdir));
+        if (!nextcomp)
+                goto err;
+
+        /* Wipe the contents of the previous component */
+        uuid_copy (gfid, mres->resolveloc.inode->gfid);
+        nfs_loc_wipe (&mres->resolveloc);
+        ret = nfs_entry_loc_fill (mres->exp->vol->itable, gfid, nextcomp,
+                                  &mres->resolveloc, NFS_RESOLVE_CREATE);
+        if ((ret < 0) && (ret != -2)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve and create "
+                        "inode: parent gfid %s, entry %s",
+                        uuid_utoa (gfid), nextcomp);
+                ret = -EFAULT;
+                goto err;
+        }
+
+        nfs_request_user_init (&nfu, mres->req);
+        ret = nfs_lookup (mres->mstate->nfsx, mres->exp->vol, &nfu,
+                          &mres->resolveloc, mnt3_resolve_subdir_cbk, mres);
+
+err:
+        return ret;
+}
+
+
+int32_t
+mnt3_resolve_subdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, dict_t *xattr,
+                         struct iatt *postparent)
+{
+        mnt3_resolve_t          *mres = NULL;
+        mountstat3              mntstat = MNT3ERR_SERVERFAULT;
+        struct nfs3_fh          fh = {{0}, };
+        int                     autharr[10];
+        int                     autharrlen = 0;
+        rpcsvc_t                *svc = NULL;
+        mountres3               res = {0, };
+        xlator_t                *mntxl = NULL;
+        char                    *path = NULL;
+
+        mres = frame->local;
+        mntxl = (xlator_t *)cookie;
+        if (op_ret == -1) {
+                gf_log (GF_NFS, GF_LOG_ERROR, "path=%s (%s)",
+                        mres->resolveloc.path, strerror (op_errno));
+                mntstat = mnt3svc_errno_to_mnterr (op_errno);
+                goto err;
+        }
+
+        inode_link (mres->resolveloc.inode, mres->resolveloc.parent,
+                    mres->resolveloc.name, buf);
+
+        nfs3_fh_build_child_fh (&mres->parentfh, buf, &fh);
+        if (strlen (mres->remainingdir) <= 0) {
+                op_ret = -1;
+                mntstat = MNT3_OK;
+                path = GF_CALLOC (PATH_MAX, sizeof (char), gf_nfs_mt_char);
+                if (!path) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation "
+                                "failed");
+                        goto err;
+                }
+                snprintf (path, PATH_MAX, "/%s%s", mres->exp->vol->name,
+                         mres->resolveloc.path);
+                mnt3svc_update_mountlist (mres->mstate, mres->req,
+                                          path);
+                GF_FREE (path);
+        } else {
+                mres->parentfh = fh;
+                op_ret = __mnt3_resolve_export_subdir_comp (mres);
+                if (op_ret < 0)
+                        mntstat = mnt3svc_errno_to_mnterr (-op_ret);
+        }
+err:
+        if (op_ret == -1) {
+                gf_log (GF_MNT, GF_LOG_DEBUG, "Mount reply status: %d",
+                        mntstat);
+                svc = rpcsvc_request_service (mres->req);
+                autharrlen = rpcsvc_auth_array (svc, mntxl->name, autharr,
+                                                10);
+
+                res = mnt3svc_set_mountres3 (mntstat, &fh, autharr, autharrlen);
+                mnt3svc_submit_reply (mres->req, (void *)&res,
+                                      (mnt3_serializer)xdr_serialize_mountres3);
+                mnt3_resolve_state_wipe (mres);
+        }
+
+        return 0;
+}
+
+
+
+/* We will always have to perform a hard lookup on all the components of a
+ * directory export for a mount request because in the mount reply we need the
+ * file handle of the directory. Our file handle creation code is designed with
+ * the assumption that to build a child file/dir fh, we'll always have the
+ * parent dir's fh available so that we may copy the hash array of the previous
+ * dir levels.
+ *
+ * Since we do not store the file handles anywhere, for every mount request we
+ * must resolve the file handles of every component so that the parent dir file
+ * of the exported directory can be built.
+ */
+int
+__mnt3_resolve_subdir (mnt3_resolve_t *mres)
+{
+        char            dupsubdir[MNTPATHLEN];
+        char            *firstcomp = NULL;
+        int             ret = -EFAULT;
+        nfs_user_t      nfu = {0, };
+        uuid_t          rootgfid = {0, };
+
+        if (!mres)
+                return ret;
+
+        firstcomp = setup_next_component (mres->remainingdir,
+                                          sizeof (mres->remainingdir),
+                                          dupsubdir, sizeof (dupsubdir));
+        if (!firstcomp)
+                goto err;
+
+        rootgfid[15] = 1;
+        ret = nfs_entry_loc_fill (mres->exp->vol->itable, rootgfid, firstcomp,
+                                  &mres->resolveloc, NFS_RESOLVE_CREATE);
+        if ((ret < 0) && (ret != -2)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve and create "
+                        "inode for volume root: %s", mres->exp->vol->name);
+                ret = -EFAULT;
+                goto err;
+        }
+
+        nfs_request_user_init (&nfu, mres->req);
+        ret = nfs_lookup (mres->mstate->nfsx, mres->exp->vol, &nfu,
+                          &mres->resolveloc, mnt3_resolve_subdir_cbk, mres);
+
+err:
+        return ret;
+}
+
+
+/**
+ * This function will verify if the client is allowed to mount
+ * the directory or not. Client's IP address will be compared with
+ * allowed IP list or range present in mnt3_export structure.
+ *
+ * @param req - RPC request. This structure contains client's IP address.
+ * @param export - mnt3_export structure. Contains allowed IP list/range.
+ *
+ * @return 0 - on Success and -EACCES on failure.
+ */
+int
+mnt3_verify_auth (rpcsvc_request_t *req, struct mnt3_export *export)
+{
+        int                     retvalue = -EACCES;
+        int                     ret = 0;
+        int                     shiftbits = 0;
+        uint32_t                ipv4netmask = 0;
+        uint32_t                routingprefix = 0;
+        struct host_auth_spec   *host = NULL;
+        struct sockaddr_in      *client_addr = NULL;
+        struct sockaddr_in      *allowed_addr = NULL;
+        struct addrinfo         *allowed_addrinfo = NULL;
+
+        /* Sanity check */
+        if ((NULL == req) ||
+            (NULL == req->trans) ||
+            (NULL == export) ||
+            (NULL == export->hostspec)) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Invalid argument");
+                return retvalue;
+        }
+
+        host = export->hostspec;
+
+
+        /* Client's IP address. */
+        client_addr = (struct sockaddr_in *)(&(req->trans->peerinfo.sockaddr));
+
+        /* Try to see if the client IP matches the allowed IP list.*/
+        while (NULL != host){
+                GF_ASSERT (host->host_addr);
+
+                if (NULL != allowed_addrinfo) {
+                        freeaddrinfo (allowed_addrinfo);
+                        allowed_addrinfo = NULL;
+                }
+
+                /* Get the addrinfo for the allowed host (host_addr). */
+                ret = getaddrinfo (host->host_addr,
+                                NULL,
+                                NULL,
+                                &allowed_addrinfo);
+                if (0 != ret){
+                        gf_log (GF_MNT, GF_LOG_ERROR, "getaddrinfo: %s\n",
+                                gai_strerror (ret));
+                        host = host->next;
+
+                        /* Failed to get IP addrinfo. Continue to check other
+                         * allowed IPs in the list.
+                         */
+                        continue;
+                }
+
+                allowed_addr = (struct sockaddr_in *)(allowed_addrinfo->ai_addr);
+
+                if (NULL == allowed_addr) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Invalid structure");
+                        break;
+                }
+
+                if (AF_INET == allowed_addr->sin_family){
+                        if (IPv4_ADDR_SIZE < host->routeprefix) {
+                                gf_log (GF_MNT, GF_LOG_ERROR, "invalid IP "
+                                        "configured for export-dir AUTH");
+                                host = host->next;
+                                continue;
+                        }
+
+                        /* -1 means no route prefix is provided. In this case
+                         * the IP should be an exact match. Which is same as
+                         * providing a route prefix of IPv4_ADDR_SIZE.
+                         */
+                        if (-1 == host->routeprefix) {
+                                routingprefix = IPv4_ADDR_SIZE;
+                        } else {
+                                routingprefix = host->routeprefix;
+                        }
+
+                        /* Create a mask from the routing prefix. User provided
+                         * CIDR address is split into IP address (host_addr) and
+                         * routing prefix (routeprefix). This CIDR address may
+                         * denote a single, distinct interface address or the
+                         * beginning address of an entire network.
+                         *
+                         * e.g. the IPv4 block 192.168.100.0/24 represents the
+                         * 256 IPv4 addresses from 192.168.100.0 to
+                         * 192.168.100.255.
+                         * Therefore to check if an IP matches 192.168.100.0/24
+                         * we should mask the IP with FFFFFF00 and compare it
+                         * with host address part of CIDR.
+                         */
+                        shiftbits = IPv4_ADDR_SIZE - routingprefix;
+                        ipv4netmask = 0xFFFFFFFFUL << shiftbits;
+
+                        /* Mask both the IPs and then check if they match
+                         * or not. */
+                        if (COMPARE_IPv4_ADDRS (allowed_addr,
+                                                client_addr,
+                                                ipv4netmask)){
+                                retvalue = 0;
+                                break;
+                        }
+                }
+
+                /* Client IP didn't match the allowed IP.
+                 * Check with the next allowed IP.*/
+               host = host->next;
+        }
+
+        if (NULL != allowed_addrinfo) {
+               freeaddrinfo (allowed_addrinfo);
+        }
+
+        return retvalue;
+}
+
+int
+mnt3_resolve_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
+                     struct mnt3_export *exp, char *subdir)
+{
+        mnt3_resolve_t  *mres = NULL;
+        int             ret = -EFAULT;
+        struct nfs3_fh  pfh = GF_NFS3FH_STATIC_INITIALIZER;
+
+        if ((!req) || (!ms) || (!exp) || (!subdir))
+                return ret;
+
+        /* Need to check AUTH */
+        if (NULL != exp->hostspec) {
+                ret = mnt3_verify_auth (req, exp);
+                if (0 != ret) {
+                        gf_log (GF_MNT,GF_LOG_ERROR,
+                                        "AUTH verification failed");
+                        return ret;
+                }
+        }
+
+        mres = GF_CALLOC (1, sizeof (mnt3_resolve_t), gf_nfs_mt_mnt3_resolve);
+        if (!mres) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                goto err;
+        }
+
+        mres->exp = exp;
+        mres->mstate = ms;
+        mres->req = req;
+        strncpy (mres->remainingdir, subdir, MNTPATHLEN);
+        if (gf_nfs_dvm_off (nfs_state (ms->nfsx)))
+                pfh = nfs3_fh_build_indexed_root_fh (mres->mstate->nfsx->children, mres->exp->vol);
+        else
+                pfh = nfs3_fh_build_uuid_root_fh (exp->volumeid);
+
+        mres->parentfh = pfh;
+        ret = __mnt3_resolve_subdir (mres);
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve export dir: %s"
+                        , mres->exp->expname);
+                GF_FREE (mres);
+        }
+
+err:
+        return ret;
+}
+
+
+int
+mnt3_resolve_export_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
+                            struct mnt3_export *exp)
+{
+        char            *volume_subdir = NULL;
+        int             ret = -EFAULT;
+
+        if ((!req) || (!ms) || (!exp))
+                return ret;
+
+        volume_subdir = __volume_subdir (exp->expname, NULL);
+        if (!volume_subdir)
+                goto err;
+
+        ret = mnt3_resolve_subdir (req, ms, exp, volume_subdir);
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve export dir: %s"
+                        , exp->expname);
+                goto err;
+        }
+
+err:
+        return ret;
+}
+
+
+int
+mnt3svc_mount (rpcsvc_request_t *req, struct mount3_state *ms,
+               struct mnt3_export *exp)
+{
+        int     ret = -EFAULT;
+
+        if ((!req) || (!ms) || (!exp))
+                return ret;
+
+        if (exp->exptype == MNT3_EXPTYPE_VOLUME)
+                ret = mnt3svc_volume_mount (req, ms, exp);
+        else if (exp->exptype == MNT3_EXPTYPE_DIR)
+                ret = mnt3_resolve_export_subdir (req, ms, exp);
+
+        return ret;
+}
+
+
+/* mnt3_mntpath_to_xlator sets this to 1 if the mount is for a full
+* volume or 2 for a subdir in the volume.
+*/
+struct mnt3_export *
+mnt3_mntpath_to_export (struct mount3_state *ms, char *dirpath)
+{
+        struct mnt3_export      *exp = NULL;
+        struct mnt3_export      *found = NULL;
+
+        if ((!ms) || (!dirpath))
+                return NULL;
+
+        LOCK (&ms->mountlock);
+        list_for_each_entry (exp, &ms->exportlist, explist) {
+
+                /* Search for the an exact match with the volume */
+                if (mnt3_match_dirpath_export (exp->expname, dirpath)) {
+                        found = exp;
+                        gf_log (GF_MNT, GF_LOG_DEBUG, "Found export volume: "
+                                "%s", exp->vol->name);
+                        goto foundexp;
+                }
+        }
+
+        gf_log (GF_MNT, GF_LOG_DEBUG, "Export not found");
+foundexp:
+        UNLOCK (&ms->mountlock);
+        return found;
+}
+
+
+int
+mnt3_check_client_net (struct mount3_state *ms, rpcsvc_request_t *req,
+                       xlator_t *targetxl)
+{
+
+        rpcsvc_t                *svc = NULL;
+        rpc_transport_t         *trans = NULL;
+        struct sockaddr_storage sastorage = {0,};
+        char                    peer[RPCSVC_PEER_STRLEN] = {0,};
+        int                     ret = -1;
+
+        if ((!ms) || (!req) || (!targetxl))
+                return -1;
+
+        svc = rpcsvc_request_service (req);
+
+        trans = rpcsvc_request_transport (req);
+        ret = rpcsvc_transport_peeraddr (trans, peer, RPCSVC_PEER_STRLEN,
+                                         &sastorage, sizeof (sastorage));
+        if (ret != 0) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to get peer addr: %s",
+                        gai_strerror (ret));
+        }
+
+        ret = rpcsvc_auth_check (svc, targetxl->name, trans);
+        if (ret == RPCSVC_AUTH_REJECT) {
+                gf_log (GF_MNT, GF_LOG_INFO, "Peer %s  not allowed", peer);
+                goto err;
+        }
+
+        ret = rpcsvc_transport_privport_check (svc, targetxl->name,
+                                               rpcsvc_request_transport (req));
+        if (ret == RPCSVC_AUTH_REJECT) {
+                gf_log (GF_MNT, GF_LOG_INFO, "Peer %s rejected. Unprivileged "
+                        "port not allowed", peer);
+                goto err;
+        }
+
+        ret = 0;
+err:
+        return ret;
+}
+
+
+int
+mnt3_parse_dir_exports (rpcsvc_request_t *req, struct mount3_state *ms,
+                        char *subdir)
+{
+        char                    volname[1024];
+        struct mnt3_export      *exp = NULL;
+        char                    *volname_ptr = NULL;
+        int                     ret = -1;
+
+        if ((!ms) || (!subdir))
+                return -1;
+
+        volname_ptr = volname;
+        subdir = __volume_subdir (subdir, &volname_ptr);
+        if (!subdir)
+                goto err;
+
+        exp = mnt3_mntpath_to_export (ms, volname);
+        if (!exp)
+                goto err;
+
+        ret = mnt3_resolve_subdir (req, ms, exp, subdir);
+        if (ret < 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve export dir: %s"
+                        , subdir);
+                goto err;
+        }
+
+err:
+        return ret;
+}
+
+
+int
+mnt3_find_export (rpcsvc_request_t *req, char *path, struct mnt3_export **e)
+{
+        int                     ret = -EFAULT;
+        struct mount3_state     *ms = NULL;
+        struct mnt3_export      *exp = NULL;
+
+        if ((!req) || (!path) || (!e))
+                return -1;
+
+        ms = (struct mount3_state *) rpcsvc_request_program_private (req);
+        if (!ms) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Mount state not present");
+                rpcsvc_request_seterr (req, SYSTEM_ERR);
+                goto err;
+        }
+
+        gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s", path);
+        exp = mnt3_mntpath_to_export (ms, path);
+        if (exp) {
+                ret = 0;
+                *e = exp;
+                goto err;
+        }
+
+        if (!gf_mnt3_export_dirs(ms)) {
+                ret = -1;
+                goto err;
+        }
+
+        ret = mnt3_parse_dir_exports (req, ms, path);
+        if (ret == 0) {
+                ret = -2;
+                goto err;
+        }
+
+err:
+        return ret;
+}
+
+
+int
+mnt3svc_mnt (rpcsvc_request_t *req)
+{
+        struct iovec            pvec = {0, };
+        char                    path[MNTPATHLEN];
+        int                     ret = -1;
+        struct mount3_state     *ms = NULL;
+        mountstat3              mntstat = MNT3ERR_SERVERFAULT;
+        struct mnt3_export      *exp = NULL;
+        struct nfs_state        *nfs = NULL;
+
+        if (!req)
+                return -1;
+
+        pvec.iov_base = path;
+        pvec.iov_len = MNTPATHLEN;
+        ret = xdr_to_mountpath (pvec, req->msg[0]);
+        if (ret == -1) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to decode args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
+        if (!ms) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Mount state not present");
+                rpcsvc_request_seterr (req, SYSTEM_ERR);
+                ret = -1;
+                goto rpcerr;
+        }
+
+        ret = 0;
+        nfs = (struct nfs_state *)ms->nfsx->private;
+        gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s", path);
+        ret = mnt3_find_export (req, path, &exp);
+        if (ret == -2) {
+                ret = 0;
+                goto rpcerr;
+        } else if (ret < 0) {
+                ret = -1;
+                mntstat = MNT3ERR_NOENT;
+                goto mnterr;
+        }
+
+        if (!nfs_subvolume_started (nfs, exp->vol)) {
+                gf_log (GF_MNT, GF_LOG_DEBUG, "Volume %s not started",
+                        exp->vol->name);
+                ret = -1;
+                mntstat = MNT3ERR_NOENT;
+                goto mnterr;
+        }
+
+        ret = mnt3_check_client_net (ms, req, exp->vol);
+        if (ret == RPCSVC_AUTH_REJECT) {
+                mntstat = MNT3ERR_ACCES;
+                gf_log (GF_MNT, GF_LOG_DEBUG, "Client mount not allowed");
+                ret = -1;
+                goto mnterr;
+        }
+
+        ret = mnt3svc_mount (req, ms, exp);
+        if (ret < 0)
+                mntstat = mnt3svc_errno_to_mnterr (-ret);
+mnterr:
+        if (ret < 0) {
+                mnt3svc_mnt_error_reply (req, mntstat);
+                ret = 0;
+        }
+
+rpcerr:
+        return ret;
+}
+
+
+int
+mnt3svc_null (rpcsvc_request_t *req)
+{
+        struct iovec    dummyvec = {0, };
+
+        if (!req) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Got NULL request!");
+                return 0;
+        }
+        rpcsvc_submit_generic (req, &dummyvec, 1,  NULL, 0, NULL);
+        return 0;
+}
+
+
+mountlist
+__build_mountlist (struct mount3_state *ms, int *count)
+{
+        struct mountbody        *mlist = NULL;
+        struct mountbody        *prev = NULL;
+        struct mountbody        *first = NULL;
+        size_t                  namelen = 0;
+        int                     ret = -1;
+        struct mountentry       *me = NULL;
+
+        if ((!ms) || (!count))
+                return NULL;
+
+        /* read rmtab, other peers might have updated it */
+        mount_read_rmtab(ms);
+
+        *count = 0;
+        gf_log (GF_MNT, GF_LOG_DEBUG, "Building mount list:");
+        list_for_each_entry (me, &ms->mountlist, mlist) {
+                namelen = strlen (me->exname);
+                mlist = GF_CALLOC (1, sizeof (*mlist), gf_nfs_mt_mountbody);
+                if (!mlist) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
+                                " failed");
+                        goto free_list;
+                }
+                if (!first)
+                        first = mlist;
+
+                mlist->ml_directory = GF_CALLOC (namelen + 2, sizeof (char),
+                                                 gf_nfs_mt_char);
+                if (!mlist->ml_directory) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
+                                " failed");
+                        goto free_list;
+                }
+
+                strcpy (mlist->ml_directory, me->exname);
+
+                namelen = strlen (me->hostname);
+                mlist->ml_hostname = GF_CALLOC (namelen + 2, sizeof (char),
+                                                gf_nfs_mt_char);
+                if (!mlist->ml_hostname) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
+                                " failed");
+                        goto free_list;
+                }
+
+                strcat (mlist->ml_hostname, me->hostname);
+
+                gf_log (GF_MNT, GF_LOG_DEBUG, "mount entry: dir: %s, host: %s",
+                        mlist->ml_directory, mlist->ml_hostname);
+                if (prev) {
+                        prev->ml_next = mlist;
+                        prev = mlist;
+                } else
+                        prev = mlist;
+
+                (*count)++;
+        }
+
+        ret = 0;
+
+free_list:
+        if (ret == -1) {
+                xdr_free_mountlist (first);
+                first = NULL;
+        }
+
+        return first;
+}
+
+
+mountlist
+mnt3svc_build_mountlist (struct mount3_state *ms, int *count)
+{
+        struct mountbody        *first = NULL;
+
+        LOCK (&ms->mountlock);
+        {
+                first = __build_mountlist (ms, count);
+        }
+        UNLOCK (&ms->mountlock);
+
+        return first;
+}
+
+
+int
+mnt3svc_dump (rpcsvc_request_t *req)
+{
+        int                     ret = -1;
+        struct mount3_state     *ms = NULL;
+        mountlist               mlist;
+        mountstat3              mstat = 0;
+        mnt3_serializer         sfunc = NULL;
+        void                    *arg = NULL;
+
+
+        if (!req)
+                return -1;
+
+        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
+        if (!ms) {
+                rpcsvc_request_seterr (req, SYSTEM_ERR);
+                goto rpcerr;
+        }
+
+        sfunc = (mnt3_serializer)xdr_serialize_mountlist;
+        mlist = mnt3svc_build_mountlist (ms, &ret);
+        arg = &mlist;
+
+        if (!mlist) {
+                if (ret != 0) {
+                        rpcsvc_request_seterr (req, SYSTEM_ERR);
+                        ret = -1;
+                        goto rpcerr;
+                } else {
+                        arg = &mstat;
+                        sfunc = (mnt3_serializer)xdr_serialize_mountstat3;
+                }
+        }
+
+        mnt3svc_submit_reply (req, arg, sfunc);
+
+        xdr_free_mountlist (mlist);
+        ret = 0;
+
+rpcerr:
+        return ret;
+}
+
+
+int
+mnt3svc_umount (struct mount3_state *ms, char *dirpath, char *hostname)
+{
+        struct mountentry       *me = NULL;
+        int                     ret = -1;
+        gf_store_handle_t       *sh = NULL;
+        struct nfs_state        *nfs = NULL;
+
+        if ((!ms) || (!dirpath) || (!hostname))
+                return -1;
+
+        nfs = (struct nfs_state *)ms->nfsx->private;
+
+        ret = gf_store_handle_new (nfs->rmtab, &sh);
+        if (ret) {
+                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
+                        nfs->rmtab);
+                return 0;
+        }
+
+        ret = gf_store_lock (sh);
+        if (ret) {
+                goto out_free;
+        }
+
+        LOCK (&ms->mountlock);
+        {
+                __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
+                if (list_empty (&ms->mountlist)) {
+                        ret = 0;
+                        goto out_unlock;
+                }
+
+                ret = -1;
+                list_for_each_entry (me, &ms->mountlist, mlist) {
+                       if ((strcmp (me->exname, dirpath) == 0) &&
+                            (strcmp (me->hostname, hostname) == 0)) {
+                               ret = 0;
+                               break;
+                       }
+                }
+
+                /* Need this check here because at the end of the search me
+                 * might still be pointing to the last entry, which may not be
+                 * the one we're looking for.
+                 */
+                if (ret == -1)  {/* Not found in list. */
+                        gf_log (GF_MNT, GF_LOG_TRACE, "Export not found");
+                        goto out_unlock;
+                }
+
+                if (!me)
+                        goto out_unlock;
+
+                gf_log (GF_MNT, GF_LOG_DEBUG, "Unmounting: dir %s, host: %s",
+                        me->exname, me->hostname);
+
+                list_del (&me->mlist);
+                GF_FREE (me);
+                __mount_rewrite_rmtab (ms, sh);
+        }
+out_unlock:
+        UNLOCK (&ms->mountlock);
+        gf_store_unlock (sh);
+out_free:
+        gf_store_handle_destroy (sh);
+        return ret;
+}
+
+
+int
+mnt3svc_umnt (rpcsvc_request_t *req)
+{
+        char                    hostname[MNTPATHLEN];
+        char                    dirpath[MNTPATHLEN];
+        struct iovec            pvec = {0, };
+        int                     ret = -1;
+        struct mount3_state     *ms = NULL;
+        mountstat3              mstat = MNT3_OK;
+        char                    *colon = NULL;
+
+        if (!req)
+                return -1;
+
+        /* Remove the mount point from the exports list. */
+        pvec.iov_base = dirpath;
+        pvec.iov_len = MNTPATHLEN;
+        ret = xdr_to_mountpath (pvec, req->msg[0]);
+        if (ret == -1) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed decode args");
+                rpcsvc_request_seterr (req, GARBAGE_ARGS);
+                goto rpcerr;
+        }
+
+        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
+        if (!ms) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Mount state not present");
+                rpcsvc_request_seterr (req, SYSTEM_ERR);
+                ret = -1;
+                goto rpcerr;
+        }
+
+        ret = rpcsvc_transport_peername (req->trans, hostname, MNTPATHLEN);
+        if (ret != 0) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get remote name: %s",
+                        gai_strerror (ret));
+                goto rpcerr;
+        }
+
+        colon = strrchr (hostname, ':');
+        if (colon) {
+                *colon= '\0';
+        }
+        gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s, hostname: %s", dirpath,
+                hostname);
+        ret = mnt3svc_umount (ms, dirpath, hostname);
+
+        if (ret == -1) {
+                ret = 0;
+                mstat = MNT3ERR_NOENT;
+        }
+        /* FIXME: also take care of the corner case where the
+         * client was resolvable at mount but not at the umount - vice-versa.
+         */
+        mnt3svc_submit_reply (req, &mstat,
+                              (mnt3_serializer)xdr_serialize_mountstat3);
+
+rpcerr:
+        return ret;
+}
+
+
+int
+__mnt3svc_umountall (struct mount3_state *ms)
+{
+        struct mountentry       *me = NULL;
+        struct mountentry       *tmp = NULL;
+
+        if (!ms)
+                return -1;
+
+        if (list_empty (&ms->mountlist))
+                return 0;
+
+        list_for_each_entry_safe (me, tmp, &ms->mountlist, mlist) {
+                list_del (&me->mlist);
+                GF_FREE (me);
+        }
+
+        return 0;
+}
+
+
+int
+mnt3svc_umountall (struct mount3_state *ms)
+{
+        int     ret = -1;
+        if (!ms)
+                return -1;
+
+        LOCK (&ms->mountlock);
+        {
+               ret = __mnt3svc_umountall (ms);
+        }
+        UNLOCK (&ms->mountlock);
+
+        return ret;
+}
+
+
+int
+mnt3svc_umntall (rpcsvc_request_t *req)
+{
+        int                     ret = RPCSVC_ACTOR_ERROR;
+        struct mount3_state     *ms = NULL;
+        mountstat3              mstat = MNT3_OK;
+
+        if (!req)
+                return ret;
+
+        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
+        if (!ms) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Mount state not present");
+                rpcsvc_request_seterr (req, SYSTEM_ERR);
+                goto rpcerr;
+        }
+
+        mnt3svc_umountall (ms);
+        mnt3svc_submit_reply (req, &mstat,
+                              (mnt3_serializer)xdr_serialize_mountstat3);
+
+        ret = RPCSVC_ACTOR_SUCCESS;
+rpcerr:
+        return ret;
+}
+
+
+exports
+mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
+{
+        struct exportnode       *elist = NULL;
+        struct exportnode       *prev = NULL;
+        struct exportnode       *first = NULL;
+        size_t                  namelen = 0;
+        int                     ret = -1;
+        char                    *addrstr = NULL;
+        struct mnt3_export      *ent = NULL;
+        struct nfs_state        *nfs = NULL;
+
+        if ((!ms) || (!svc))
+                return NULL;
+
+        nfs = (struct nfs_state *)ms->nfsx->private;
+        if (!nfs)
+                return NULL;
+
+        LOCK (&ms->mountlock);
+        list_for_each_entry(ent, &ms->exportlist, explist) {
+
+                /* If volume is not started yet, do not list it for tools like
+                 * showmount.
+                 */
+                if (!nfs_subvolume_started (nfs, ent->vol))
+                        continue;
+
+                namelen = strlen (ent->expname) + 1;
+                elist = GF_CALLOC (1, sizeof (*elist), gf_nfs_mt_exportnode);
+                if (!elist) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
+                                " failed");
+                        goto free_list;
+                }
+                 if (!first)
+                         first = elist;
+                elist->ex_dir = GF_CALLOC (namelen + 2, sizeof (char),
+                                           gf_nfs_mt_char);
+                if (!elist->ex_dir) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
+                                " failed");
+                        goto free_list;
+                }
+                strcpy (elist->ex_dir, ent->expname);
+
+                addrstr = rpcsvc_volume_allowed (svc->options,
+                                                 ent->vol->name);
+                elist->ex_groups = GF_CALLOC (1, sizeof (struct groupnode),
+                                              gf_nfs_mt_groupnode);
+                if (!elist->ex_groups) {
+                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
+                                " failed");
+                        goto free_list;
+                }
+                /*This check has to be done after checking
+                 * elist->ex_groups allocation check to avoid resource leak;
+                */
+                if (addrstr)
+                        addrstr = gf_strdup (addrstr);
+                else
+                        addrstr = gf_strdup ("No Access");
+
+                if (!addrstr) {
+                        goto free_list;
+                }
+                elist->ex_groups->gr_name = addrstr;
+                if (prev) {
+                        prev->ex_next = elist;
+                        prev = elist;
+                } else
+                        prev = elist;
+        }
+
+        ret = 0;
+
+free_list:
+        UNLOCK (&ms->mountlock);
+        if (ret == -1) {
+                xdr_free_exports_list (first);
+                first = NULL;
+        }
+
+        return first;
+}
+
+
+int
+mnt3svc_export (rpcsvc_request_t *req)
+{
+        struct mount3_state     *ms = NULL;
+        exports                 elist = NULL;
+        int                     ret = -1;
+
+        if (!req)
+                return -1;
+
+        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
+        if (!ms) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "mount state not found");
+                rpcsvc_request_seterr (req, SYSTEM_ERR);
+                goto err;
+        }
+
+        /* Using the children translator names, build the export list */
+        elist = mnt3_xlchildren_to_exports (rpcsvc_request_service (req),
+                                            ms);
+        /* Do not return error when exports list is empty. An exports list can
+         * be empty when no subvolumes have come up. No point returning error
+         * and confusing the user.
+        if (!elist) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to build exports list");
+                nfs_rpcsvc_request_seterr (req, SYSTEM_ERR);
+                goto err;
+        }
+        */
+
+        /* Note how the serializer is passed to the generic reply function. */
+        mnt3svc_submit_reply (req, &elist,
+                              (mnt3_serializer)xdr_serialize_exports);
+
+        xdr_free_exports_list (elist);
+        ret = 0;
+err:
+        return ret;
+}
+
+/* just declaring, definition is way down below */
+rpcsvc_program_t        mnt3prog;
+
+/* nfs3_rootfh used by mount3udp thread needs to access mount3prog.private
+ * directly as we don't have nfs xlator pointer to dereference it. But thats OK
+ */
+
+struct nfs3_fh *
+nfs3_rootfh (char* path)
+{
+        struct mount3_state     *ms = NULL;
+        struct nfs3_fh          *fh = NULL;
+        struct mnt3_export      *exp = NULL;
+        inode_t                 *inode = NULL;
+        char                    *tmp = NULL;
+
+        ms = mnt3prog.private;
+        exp = mnt3_mntpath_to_export (ms, path);
+        if (exp == NULL)
+                goto err;
+
+        tmp = (char *)path;
+        tmp = strchr (tmp, '/');
+        if (tmp == NULL)
+                tmp = "/";
+
+        inode = inode_from_path (exp->vol->itable, tmp);
+        if (inode == NULL)
+                goto err;
+
+        fh = GF_CALLOC (1, sizeof(*fh), gf_nfs_mt_nfs3_fh);
+        if (fh == NULL)
+                goto err;
+        nfs3_build_fh (inode, exp->volumeid, fh);
+
+err:
+        if (inode)
+                inode_unref (inode);
+        return fh;
+}
+
+int
+mount3udp_add_mountlist (char *host, dirpath *expname)
+{
+        struct mountentry       *me = NULL;
+        struct mount3_state     *ms = NULL;
+        char                    *export = NULL;
+
+        ms = mnt3prog.private;
+        me = GF_CALLOC (1, sizeof (*me), gf_nfs_mt_mountentry);
+        if (!me)
+                return -1;
+        export = (char *)expname;
+        while (*export == '/')
+                export++;
+
+        strncpy (me->exname, export, MNTPATHLEN);
+        strncpy (me->hostname, host, MNTPATHLEN);
+        INIT_LIST_HEAD (&me->mlist);
+        LOCK (&ms->mountlock);
+        {
+                list_add_tail (&me->mlist, &ms->mountlist);
+                mount_rewrite_rmtab(ms, NULL);
+        }
+        UNLOCK (&ms->mountlock);
+        return 0;
+}
+
+int
+mount3udp_delete_mountlist (char *hostname, dirpath *expname)
+{
+        struct mount3_state     *ms = NULL;
+        char                    *export = NULL;
+
+        ms = mnt3prog.private;
+        export = (char *)expname;
+        while (*export == '/')
+                export++;
+        mnt3svc_umount (ms, export, hostname);
+        return 0;
+}
+
+/**
+ * This function will parse the hostip (IP addres, IP range, or hostname)
+ * and fill the host_auth_spec structure.
+ *
+ * @param hostspec - struct host_auth_spec
+ * @param hostip   - IP address, IP range (CIDR format) or hostname
+ *
+ * @return 0 - on success and -1 on failure
+ */
+int
+mnt3_export_fill_hostspec (struct host_auth_spec* hostspec, const char* hostip)
+{
+        char *ipdupstr = NULL;
+        char *savptr = NULL;
+        char *ip = NULL;
+        char *token = NULL;
+        int  ret = -1;
+
+        /* Create copy of the string so that the source won't change
+         */
+        ipdupstr = gf_strdup (hostip);
+        if (NULL == ipdupstr) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                goto err;
+        }
+
+        ip = strtok_r (ipdupstr, "/", &savptr);
+        hostspec->host_addr = gf_strdup (ip);
+        if (NULL == hostspec->host_addr) {
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                goto err;
+        }
+
+        /* Check if the IP is in <IP address> / <Range> format.
+         * If yes, then strip the range and store it separately.
+         */
+        token = strtok_r (NULL, "/", &savptr);
+
+        if (NULL == token) {
+              hostspec->routeprefix = -1;
+        } else {
+              hostspec->routeprefix = atoi (token);
+        }
+
+        // success
+        ret = 0;
+err:
+        if (NULL != ipdupstr) {
+                GF_FREE (ipdupstr);
+        }
+        return ret;
+}
+
+
+/**
+ * This function will parse the AUTH parameter passed along with
+ * "export-dir" option. If AUTH parameter is present then it will be
+ * stripped from exportpath and stored in mnt3_export (exp) structure.
+ *
+ * @param exp - mnt3_export structure. Holds information needed for mount.
+ * @param exportpath - Value of "export-dir" key. Holds both export path
+ *                     and AUTH parameter for the path.
+ *                     exportpath format: <abspath>[(hostdesc[|hostspec|...])]
+ *
+ * @return This function will return 0 on success and -1 on failure.
+ */
+int
+mnt3_export_parse_auth_param (struct mnt3_export* exp, char* exportpath)
+{
+        char *token = NULL;
+        char *savPtr = NULL;
+        char *hostip = NULL;
+        struct host_auth_spec *host = NULL;
+        int ret = 0;
+
+        /* Using exportpath directly in strtok_r because we want
+         * to strip off AUTH parameter from exportpath. */
+        token = strtok_r (exportpath, "(", &savPtr);
+
+        /* Get the next token, which will be the AUTH parameter. */
+        token = strtok_r (NULL, ")", &savPtr);
+
+        if (NULL == token) {
+                /* If AUTH is not present then we should return success. */
+                return 0;
+        }
+
+        /* Free any previously allocated hostspec structure. */
+        if (NULL != exp->hostspec) {
+                GF_FREE (exp->hostspec);
+                exp->hostspec = NULL;
+        }
+
+        exp->hostspec = GF_CALLOC (1,
+                        sizeof (*(exp->hostspec)),
+                        gf_nfs_mt_auth_spec);
+        if (NULL == exp->hostspec){
+                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
+                return -1;
+        }
+
+        /* AUTH parameter can have multiple entries. For each entry
+         * a host_auth_spec structure is created. */
+        host = exp->hostspec;
+
+        hostip = strtok_r (token, "|", &savPtr);
+
+        /* Parse all AUTH parameters separated by '|' */
+        while (NULL != hostip){
+                ret = mnt3_export_fill_hostspec (host, hostip);
+                if (0 != ret) {
+                        gf_log(GF_MNT, GF_LOG_WARNING,
+                                        "Failed to parse hostspec: %s", hostip);
+                        goto err;
+                }
+
+                hostip = strtok_r (NULL, "|", &savPtr);
+                if (NULL == hostip) {
+                        break;
+                }
+
+                host->next = GF_CALLOC (1, sizeof (*(host)),
+                                gf_nfs_mt_auth_spec);
+                if (NULL == host->next){
+                        gf_log (GF_MNT,GF_LOG_ERROR,
+                                        "Memory allocation failed");
+                        goto err;
+                }
+                host = host->next;
+        }
+
+        /* In case of success return from here */
+        return 0;
+err:
+        /* In case of failure free up hostspec structure.  */
+        FREE_HOSTSPEC (exp);
+
+        return -1;
+}
+
+/**
+ * exportpath will also have AUTH options (ip address, subnet address or
+ * hostname) mentioned.
+ * exportpath format: <abspath>[(hostdesc[|hostspec|...])]
+ */

path: root/libglusterfs/src/gfdb/gfdb_sqlite3_helper.c

/*
   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.

   This file is licensed to you under your choice of the GNU Lesser
   General Public License, version 3 or any later version (LGPLv3 or
   later), or the GNU General Public License, version 2 (GPLv2), in all
   cases as published by the Free Software Foundation.
*/

#include "gfdb_sqlite3_helper.h"


#define GFDB_SQL_STMT_SIZE 256

/*****************************************************************************
 *
 *                 Helper function to execute actual sql queries
 *
 *
 * ****************************************************************************/

static int
gf_sql_delete_all (gf_sql_connection_t  *sql_conn,
                  char                  *gfid,
                  gf_boolean_t          ignore_errors)
{
        int ret = -1;
        sqlite3_stmt *delete_file_stmt = NULL;
        sqlite3_stmt *delete_link_stmt = NULL;
        char *delete_link_str = "DELETE FROM "
                           GF_FILE_LINK_TABLE
                           " WHERE GF_ID = ? ;";
        char *delete_file_str = "DELETE FROM "
                           GF_FILE_TABLE
                           " WHERE GF_ID = ? ;";

        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);

        /*
         * Delete all links associated with this GFID
         *
         * */
        /*Prepare statement for delete all links*/
        ret = sqlite3_prepare(sql_conn->sqlite3_db_conn, delete_link_str, -1,
                                &delete_link_stmt, 0);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_PREPARE_FAILED, "Failed preparing delete "
                        "statment %s : %s", delete_link_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind gfid*/
        ret = sqlite3_bind_text (delete_link_stmt, 1, gfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }


        /*Execute the prepare statement*/
        if (sqlite3_step (delete_link_stmt) != SQLITE_DONE) {
                gf_msg (GFDB_STR_SQLITE3,
                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
                        LG_MSG_EXEC_FAILED,
                        "Failed executing the prepared stmt %s : %s",
                        delete_link_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }


        /*
         * Delete entry from file table associated with this GFID
         *
         * */
        /*Prepare statement for delete all links*/
        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, delete_file_str, -1,
                                &delete_file_stmt, 0);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_PREPARE_FAILED, "Failed preparing delete "
                        "statment %s : %s", delete_file_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind gfid*/
        ret = sqlite3_bind_text (delete_file_stmt, 1, gfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Execute the prepare statement*/
        if (sqlite3_step (delete_file_stmt) != SQLITE_DONE) {
                gf_msg (GFDB_STR_SQLITE3,
                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
                        LG_MSG_EXEC_FAILED,
                        "Failed executing the prepared stmt %s : %s",
                        delete_file_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

out:
        /*Free prepared statement*/
        sqlite3_finalize (delete_file_stmt);
        sqlite3_finalize (delete_link_stmt);
        return ret;
}

static int
gf_sql_delete_link (gf_sql_connection_t  *sql_conn,
                   char                 *gfid,
                   char                 *pargfid,
                   char                 *basename,
                   gf_boolean_t         ignore_errors)
{
        int ret = -1;
        sqlite3_stmt *delete_stmt = NULL;
        char *delete_str = "DELETE FROM "
                           GF_FILE_LINK_TABLE
                           " WHERE GF_ID = ? AND GF_PID = ?"
                           " AND FNAME = ?;";

        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pargfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basename, out);

        /*Prepare statement*/
        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, delete_str, -1,
                                &delete_stmt, 0);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_PREPARE_FAILED, "Failed preparing delete "
                        "statment %s : %s", delete_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind gfid*/
        ret = sqlite3_bind_text (delete_stmt, 1, gfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED,
                        "Failed binding gfid %s : %s", gfid,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind pargfid*/
        ret = sqlite3_bind_text (delete_stmt, 2, pargfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding parent gfid %s "
                        ": %s", pargfid,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind basename*/
        ret = sqlite3_bind_text (delete_stmt, 3, basename, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding basename %s : "
                        "%s", basename,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Execute the prepare statement*/
        if (sqlite3_step(delete_stmt) != SQLITE_DONE) {
                gf_msg (GFDB_STR_SQLITE3,
                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
                        LG_MSG_EXEC_FAILED,
                        "Failed executing the prepared stmt %s : %s",
                        delete_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }


        ret = 0;
out:
        /*Free prepared statement*/
        sqlite3_finalize (delete_stmt);
        return ret;
}



static int
gf_sql_update_link_flags (gf_sql_connection_t  *sql_conn,
                   char                 *gfid,
                   char                 *pargfid,
                   char                 *basename,
                   int                  update_flag,
                   gf_boolean_t         is_update_or_delete,
                   gf_boolean_t         ignore_errors)
{
        int ret = -1;
        sqlite3_stmt *update_stmt = NULL;
        char *update_column = NULL;
        char update_str[1024] = "";


        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pargfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basename, out);

        update_column = (is_update_or_delete) ? "LINK_UPDATE" : "W_DEL_FLAG";

        sprintf (update_str, "UPDATE "
                            GF_FILE_LINK_TABLE
                            " SET %s = ?"
                            " WHERE GF_ID = ? AND GF_PID = ? AND FNAME = ?;",
                            update_column);

        /*Prepare statement*/
        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, update_str, -1,
                                &update_stmt, 0);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_PREPARE_FAILED, "Failed preparing update "
                        "statment %s : %s", update_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }


        /*Bind link_update*/
        ret = sqlite3_bind_int (update_stmt, 1, update_flag);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding update_flag %d "
                        ": %s", update_flag,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind gfid*/
        ret = sqlite3_bind_text (update_stmt, 2, gfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind pargfid*/
        ret = sqlite3_bind_text (update_stmt, 3, pargfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding parent gfid %s "
                        ": %s", pargfid,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind basename*/
        ret = sqlite3_bind_text (update_stmt, 4, basename, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding basename %s : "
                        "%s", basename,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }


        /*Execute the prepare statement*/
        if (sqlite3_step(update_stmt) != SQLITE_DONE) {
                gf_msg (GFDB_STR_SQLITE3,
                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
                        LG_MSG_EXEC_FAILED,
                        "Failed executing the prepared stmt %s : %s",
                        update_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        ret = 0;
out:
        /*Free prepared statement*/
        sqlite3_finalize (update_stmt);
        return ret;
}


static int
gf_sql_insert_link (gf_sql_connection_t  *sql_conn,
                   char                 *gfid,
                   char                 *pargfid,
                   char                 *basename,
                   char                 *basepath,
                   gf_boolean_t         link_consistency,
                   gf_boolean_t         ignore_errors)
{
        int ret = -1;
        sqlite3_stmt *insert_stmt = NULL;
        char insert_str[GFDB_SQL_STMT_SIZE] = "";

        sprintf (insert_str, "INSERT INTO "
                           GF_FILE_LINK_TABLE
                           " (GF_ID, GF_PID, FNAME, FPATH,"
                           " W_DEL_FLAG, LINK_UPDATE) "
                           " VALUES (?, ?, ?, ?, 0, %d);",
                           link_consistency);

        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pargfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basename, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basepath, out);

        /*Prepare statement*/
        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, insert_str, -1,
                                &insert_stmt, 0);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_PREPARE_FAILED,
                        "Failed preparing insert "
                        "statment %s : %s", insert_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind gfid*/
        ret = sqlite3_bind_text (insert_stmt, 1, gfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED,
                        "Failed binding gfid %s : %s",
                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind pargfid*/
        ret = sqlite3_bind_text (insert_stmt, 2, pargfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
                        0, LG_MSG_BINDING_FAILED,
                        "Failed binding parent gfid %s "
                        ": %s", pargfid,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind basename*/
        ret = sqlite3_bind_text (insert_stmt, 3, basename, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
                        0, LG_MSG_BINDING_FAILED,
                        "Failed binding basename %s : %s", basename,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind basepath*/
        ret = sqlite3_bind_text (insert_stmt, 4, basepath, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED,
                        "Failed binding basepath %s : "
                        "%s", basepath,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Execute the prepare statement*/
        if (sqlite3_step (insert_stmt) != SQLITE_DONE) {
                gf_msg (GFDB_STR_SQLITE3,
                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors),
                        0, LG_MSG_EXEC_FAILED,
                        "Failed executing the prepared "
                        "stmt %s %s %s %s %s : %s",
                        gfid, pargfid, basename, basepath, insert_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        ret = 0;
out:
        /*Free prepared statement*/
        sqlite3_finalize (insert_stmt);
        return ret;
}


static int
gf_sql_update_link (gf_sql_connection_t  *sql_conn,
                   char                 *gfid,
                   char                 *pargfid,
                   char                 *basename,
                   char                 *basepath,
                   char                 *old_pargfid,
                   char                 *old_basename,
                   gf_boolean_t         link_consistency,
                   gf_boolean_t         ignore_errors)
{
        int ret = -1;
        sqlite3_stmt *insert_stmt = NULL;
        char insert_str[GFDB_SQL_STMT_SIZE] = "";

        sprintf (insert_str, "INSERT INTO "
                            GF_FILE_LINK_TABLE
                            " (GF_ID, GF_PID, FNAME, FPATH,"
                            " W_DEL_FLAG, LINK_UPDATE) "
                            " VALUES (? , ?, ?, ?, 0, %d);",
                            link_consistency);

        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, pargfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basename, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, basepath, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, old_pargfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, old_basename, out);

        /*
         *
         * Delete the old link
         *
         * */
         ret = gf_sql_delete_link (sql_conn, gfid, old_pargfid,
                                old_basename, ignore_errors);
        if (ret) {
                gf_msg (GFDB_STR_SQLITE3,
                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
                        LG_MSG_DELETE_FAILED, "Failed deleting old link");
                goto out;
        }

        /*
         *
         * insert new link
         *
         * */
        /*Prepare statement*/
        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, insert_str, -1,
                                &insert_stmt, 0);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_PREPARE_FAILED, "Failed preparing insert "
                        "statment %s : %s", insert_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind gfid*/
        ret = sqlite3_bind_text (insert_stmt, 1, gfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind new pargfid*/
        ret = sqlite3_bind_text (insert_stmt, 2, pargfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding parent gfid %s "
                        ": %s", pargfid,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind new basename*/
        ret = sqlite3_bind_text (insert_stmt, 3, basename, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding basename %s : "
                        "%s", basename,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind new basepath*/
        ret = sqlite3_bind_text (insert_stmt, 4, basepath, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding basename %s : "
                        "%s", basepath,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Execute the prepare statement*/
        if (sqlite3_step (insert_stmt) != SQLITE_DONE) {
                gf_msg (GFDB_STR_SQLITE3,
                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
                        LG_MSG_EXEC_FAILED,
                        "Failed executing the prepared stmt %s : %s",
                        insert_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }



        ret = 0;
out:
        /*Free prepared statement*/
        sqlite3_finalize (insert_stmt);
        return ret;
}

static int
gf_sql_insert_write_wind_time (gf_sql_connection_t      *sql_conn,
                                char                    *gfid,
                                gfdb_time_t             *wind_time,
                                gf_boolean_t            ignore_errors)
{
        int ret = -1;
        sqlite3_stmt *insert_stmt = NULL;
        char *insert_str = "INSERT INTO "
                           GF_FILE_TABLE
                           "(GF_ID, W_SEC, W_MSEC, UW_SEC, UW_MSEC)"
                           " VALUES (?, ?, ?, 0, 0);";

        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, wind_time, out);


        /*Prepare statement*/
        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, insert_str, -1,
                        &insert_stmt, 0);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_PREPARE_FAILED, "Failed preparing insert "
                        "statment %s : %s", insert_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind gfid*/
        ret = sqlite3_bind_text (insert_stmt, 1, gfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind wind secs*/
        ret = sqlite3_bind_int (insert_stmt, 2, wind_time->tv_sec);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding parent wind "
                        "secs %ld : %s", wind_time->tv_sec,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind wind msecs*/
        ret = sqlite3_bind_int (insert_stmt, 3, wind_time->tv_usec);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding parent wind "
                        "msecs %ld : %s", wind_time->tv_usec,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Execute the prepare statement*/
        if (sqlite3_step (insert_stmt) != SQLITE_DONE) {
                gf_msg (GFDB_STR_SQLITE3,
                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
                        LG_MSG_EXEC_FAILED,
                        "Failed executing the prepared stmt GFID:%s %s : %s",
                        gfid, insert_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        ret = 0;
out:
        /*Free prepared statement*/
        sqlite3_finalize (insert_stmt);
        return ret;
}



/*Update write/read times for both wind and unwind*/
static int
gf_update_time (gf_sql_connection_t    *sql_conn,
                char                    *gfid,
                gfdb_time_t             *update_time,
                gf_boolean_t            record_counter,
                gf_boolean_t            is_wind,
                gf_boolean_t            is_read,
                gf_boolean_t            ignore_errors)
{
        int ret = -1;
        sqlite3_stmt *update_stmt = NULL;
        char update_str[1024] = "";
        char *freq_cntr_str = NULL;

        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfid, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, update_time, out);

        /*
         * Constructing the prepare statment string.
         *
         * */
        /*For write time*/
        if (!is_read) {
                if (is_wind) {
                        /*if record counter is on*/
                        freq_cntr_str = (record_counter) ?
                        ", WRITE_FREQ_CNTR = WRITE_FREQ_CNTR + 1" : "";

                        /*Prefectly safe as we will not go array of bound*/
                        sprintf (update_str, "UPDATE "
                                GF_FILE_TABLE
                                " SET W_SEC = ?, W_MSEC = ? "
                                " %s"/*place for read freq counters*/
                                " WHERE GF_ID = ? ;", freq_cntr_str);
                } else {
                        /*Prefectly safe as we will not go array of bound*/
                        sprintf (update_str, "UPDATE "
                                GF_FILE_TABLE
                                " SET UW_SEC = ?, UW_MSEC = ? ;");
                }
        }
        /*For Read Time update*/
        else {
                if (is_wind) {
                        /*if record counter is on*/
                        freq_cntr_str = (record_counter) ?
                        ", READ_FREQ_CNTR = READ_FREQ_CNTR + 1" : "";

                        /*Prefectly safe as we will not go array of bound*/
                        sprintf (update_str, "UPDATE "
                                GF_FILE_TABLE
                                " SET W_READ_SEC = ?, W_READ_MSEC = ? "
                                " %s"/*place for read freq counters*/
                                " WHERE GF_ID = ? ;", freq_cntr_str);
                } else {
                        /*Prefectly safe as we will not go array of bound*/
                        sprintf (update_str, "UPDATE "
                                GF_FILE_TABLE
                                " SET UW_READ_SEC = ?, UW_READ_MSEC = ? ;");
                }
        }

        /*Prepare statement*/
        ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, update_str, -1,
                                &update_stmt, 0);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_PREPARE_FAILED, "Failed preparing insert "
                        "statment %s : %s", update_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind time secs*/
        ret = sqlite3_bind_int (update_stmt, 1, update_time->tv_sec);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding parent wind "
                        "secs %ld : %s", update_time->tv_sec,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind time msecs*/
        ret = sqlite3_bind_int (update_stmt, 2, update_time->tv_usec);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding parent wind "
                        "msecs %ld : %s", update_time->tv_usec,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Bind gfid*/
        ret = sqlite3_bind_text (update_stmt, 3, gfid, -1, NULL);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_BINDING_FAILED, "Failed binding gfid %s : %s",
                        gfid, sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        /*Execute the prepare statement*/
        if (sqlite3_step (update_stmt) != SQLITE_DONE) {
                gf_msg (GFDB_STR_SQLITE3,
                        _gfdb_log_level (GF_LOG_ERROR, ignore_errors), 0,
                        LG_MSG_EXEC_FAILED,
                        "Failed executing the prepared stmt %s : %s",
                        update_str,
                        sqlite3_errmsg (sql_conn->sqlite3_db_conn));
                ret = -1;
                goto out;
        }

        ret = 0;
out:
        /*Free prepared statement*/
        sqlite3_finalize (update_stmt);
        return ret;
}

/******************************************************************************
 *
 *                      Helper functions for gf_sqlite3_insert()
 *
 *
 * ****************************************************************************/

int
gf_sql_insert_wind (gf_sql_connection_t  *sql_conn,
                   gfdb_db_record_t     *gfdb_db_record)
{
        int ret                 = -1;
        gfdb_time_t *modtime    = NULL;
        char *pargfid_str       = NULL;
        char *gfid_str          = NULL;
        char *old_pargfid_str   = NULL;
        gf_boolean_t its_wind   = _gf_true;/*remains true for this function*/



        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfdb_db_record, out);


        gfid_str = gf_strdup (uuid_utoa (gfdb_db_record->gfid));
        if (!gfid_str) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_CREATE_FAILED,
                        "Creating gfid string failed.");
                goto out;
        }

        modtime = &gfdb_db_record->gfdb_wind_change_time;

        /* handle all dentry based operations */
        if (isdentryfop (gfdb_db_record->gfdb_fop_type)) {
                /*Parent GFID is always set*/
                pargfid_str = gf_strdup (uuid_utoa (gfdb_db_record->pargfid));
                if (!pargfid_str) {
                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
                                0, LG_MSG_CREATE_FAILED, "Creating gfid string "
                                "failed.");
                        goto out;
                }

                /* handle create, mknod */
                if (isdentrycreatefop (gfdb_db_record->gfdb_fop_type)) {
                        /*insert link*/
                        ret = gf_sql_insert_link(sql_conn,
                                        gfid_str, pargfid_str,
                                        gfdb_db_record->file_name,
                                        gfdb_db_record->file_path,
                                        gfdb_db_record->link_consistency,
                                        gfdb_db_record->ignore_errors);
                        if (ret) {
                                gf_msg (GFDB_STR_SQLITE3,
                                        _gfdb_log_level (GF_LOG_ERROR,
                                                gfdb_db_record->ignore_errors),
                                        0,
                                        LG_MSG_INSERT_FAILED, "Failed "
                                        "inserting link in DB");
                                goto out;
                        }
                        gfdb_db_record->islinkupdate = gfdb_db_record->
                                                        link_consistency;

                        /*
                         * Only for create/mknod insert wind time
                         * for the first time
                         * */
                        ret = gf_sql_insert_write_wind_time (sql_conn, gfid_str,
                                        modtime, gfdb_db_record->ignore_errors);
                        if (ret) {
                                gf_msg (GFDB_STR_SQLITE3,
                                        _gfdb_log_level (GF_LOG_ERROR,
                                                gfdb_db_record->ignore_errors),
                                        0, LG_MSG_INSERT_FAILED,
                                        "Failed inserting wind time in DB");
                                goto out;
                        }
                        goto out;
                }
                /*handle rename, link */
                else {
                        /*rename*/
                        if (strlen (gfdb_db_record->old_file_name) != 0) {
                                old_pargfid_str = gf_strdup (uuid_utoa (
                                                gfdb_db_record->old_pargfid));
                                if (!old_pargfid_str) {
                                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
                                                0, LG_MSG_CREATE_FAILED,
                                                "Creating gfid string failed.");
                                        goto out;
                                }
                                ret = gf_sql_update_link (sql_conn, gfid_str,
                                                pargfid_str,
                                                gfdb_db_record->file_name,
                                                gfdb_db_record->file_path,
                                                old_pargfid_str,
                                                gfdb_db_record->old_file_name,
                                                gfdb_db_record->
                                                        link_consistency,
                                                gfdb_db_record->ignore_errors);
                                if (ret) {
                                        gf_msg (GFDB_STR_SQLITE3,
                                                _gfdb_log_level (GF_LOG_ERROR,
                                                gfdb_db_record->ignore_errors),
                                                0, LG_MSG_UPDATE_FAILED,
                                                "Failed updating link");
                                        goto out;
                                }
                                gfdb_db_record->islinkupdate = gfdb_db_record->
                                                        link_consistency;
                        }
                        /*link*/
                        else {
                                ret = gf_sql_insert_link (sql_conn,
                                                gfid_str, pargfid_str,
                                                gfdb_db_record->file_name,
                                                gfdb_db_record->file_path,
                                                gfdb_db_record->
                                                        link_consistency,
                                                gfdb_db_record->ignore_errors);
                                if (ret) {
                                        gf_msg (GFDB_STR_SQLITE3,
                                                _gfdb_log_level (GF_LOG_ERROR,
                                                gfdb_db_record->ignore_errors),
                                                0, LG_MSG_INSERT_FAILED,
                                                "Failed inserting link in DB");
                                        goto out;
                                }
                                gfdb_db_record->islinkupdate = gfdb_db_record->
                                                        link_consistency;
                        }
                }
        }

        /* update times only when said!*/
        if (gfdb_db_record->do_record_times) {
                /*All fops update times read or write*/
                ret = gf_update_time (sql_conn, gfid_str, modtime,
                                gfdb_db_record->do_record_counters,
                                its_wind,
                                isreadfop (gfdb_db_record->gfdb_fop_type),
                                gfdb_db_record->ignore_errors);
                if (ret) {
                        gf_msg (GFDB_STR_SQLITE3,
                                _gfdb_log_level (GF_LOG_ERROR,
                                gfdb_db_record->ignore_errors), 0,
                                LG_MSG_UPDATE_FAILED, "Failed update wind time"
                                " in DB");
                        goto out;
                }
        }

        ret = 0;
out:
        GF_FREE (gfid_str);
        GF_FREE (pargfid_str);
        GF_FREE (old_pargfid_str);
        return ret;
}




int
gf_sql_insert_unwind (gf_sql_connection_t  *sql_conn,
                     gfdb_db_record_t     *gfdb_db_record)
{

        int ret = -1;
        gfdb_time_t *modtime    = NULL;
        gf_boolean_t its_wind   = _gf_true;/*remains true for this function*/
        char *gfid_str = NULL;
        char *pargfid_str = NULL;

        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfdb_db_record, out);

        gfid_str = gf_strdup (uuid_utoa(gfdb_db_record->gfid));
        if (!gfid_str) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_CREATE_FAILED, "Creating gfid string failed.");
                goto out;
        }

        /*Only update if recording unwind is set*/
        if (gfdb_db_record->do_record_times &&
                gfdb_db_record->do_record_uwind_time) {
                modtime = &gfdb_db_record->gfdb_unwind_change_time;
                ret = gf_update_time (sql_conn, gfid_str, modtime,
                        gfdb_db_record->do_record_counters,
                        (!its_wind),
                        isreadfop (gfdb_db_record->gfdb_fop_type),
                        gfdb_db_record->ignore_errors);
                if (ret) {
                        gf_msg (GFDB_STR_SQLITE3,
                                _gfdb_log_level (GF_LOG_ERROR,
                                        gfdb_db_record->ignore_errors),
                                0, LG_MSG_UPDATE_FAILED, "Failed update unwind "
                                "time in DB");
                        goto out;
                }
        }

        /*For link creation and changes we use link updated*/
        if (gfdb_db_record->islinkupdate &&
                isdentryfop(gfdb_db_record->gfdb_fop_type)) {

                pargfid_str = gf_strdup(uuid_utoa(gfdb_db_record->pargfid));
                if (!pargfid_str) {
                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
                                0, LG_MSG_CREATE_FAILED,
                                "Creating pargfid_str string failed.");
                        goto out;
                }

                ret = gf_sql_update_link_flags (sql_conn, gfid_str, pargfid_str,
                                        gfdb_db_record->file_name, 0, _gf_true,
                                        gfdb_db_record->ignore_errors);
                if (ret) {
                        gf_msg (GFDB_STR_SQLITE3,
                                _gfdb_log_level (GF_LOG_ERROR,
                                        gfdb_db_record->ignore_errors),
                                0, LG_MSG_UPDATE_FAILED,
                                "Failed updating link flags in unwind");
                        goto out;
                }
        }

        ret = 0;
out:
        GF_FREE (gfid_str);
        GF_FREE (pargfid_str);
        return ret;
}


int
gf_sql_update_delete_wind (gf_sql_connection_t  *sql_conn,
                          gfdb_db_record_t     *gfdb_db_record)
{
        int ret = -1;
        char *gfid_str          = NULL;
        char *pargfid_str       = NULL;

        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfdb_db_record, out);

        gfid_str = gf_strdup (uuid_utoa(gfdb_db_record->gfid));
        if (!gfid_str) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_CREATE_FAILED,
                        "Creating gfid string failed.");
                goto out;
        }

        pargfid_str = gf_strdup (uuid_utoa(gfdb_db_record->pargfid));
        if (!pargfid_str) {
                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
                                0, LG_MSG_CREATE_FAILED, "Creating pargfid_str "
                                "string failed.");
                        goto out;
        }

        if (gfdb_db_record->link_consistency) {
                ret = gf_sql_update_link_flags (sql_conn, gfid_str, pargfid_str,
                                        gfdb_db_record->file_name, 1,
                                        _gf_false,
                                        gfdb_db_record->ignore_errors);
                if (ret) {
                        gf_msg (GFDB_STR_SQLITE3,
                                _gfdb_log_level (GF_LOG_ERROR,
                                        gfdb_db_record->ignore_errors),
                                0, LG_MSG_UPDATE_FAILED,
                                "Failed updating link flags in wind");
                        goto out;
                }
        }

        ret = 0;
out:
        GF_FREE (gfid_str);
        GF_FREE (pargfid_str);
        return ret;
}

int
gf_sql_delete_unwind (gf_sql_connection_t  *sql_conn,
                          gfdb_db_record_t     *gfdb_db_record)
{
        int ret = -1;
        char *gfid_str = NULL;
        char *pargfid_str = NULL;
        gfdb_time_t *modtime    = NULL;

        CHECK_SQL_CONN (sql_conn, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, gfdb_db_record, out);

        gfid_str = gf_strdup (uuid_utoa(gfdb_db_record->gfid));
        if (!gfid_str) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_CREATE_FAILED,
                        "Creating gfid string failed.");
                goto out;
        }

        /*Nuke all the entries for this GFID from DB*/
        if (gfdb_db_record->gfdb_fop_path == GFDB_FOP_UNDEL_ALL) {
                gf_sql_delete_all (sql_conn, gfid_str,
                                gfdb_db_record->ignore_errors);
        }
        /*Remove link entries only*/
        else if (gfdb_db_record->gfdb_fop_path == GFDB_FOP_UNDEL) {

                pargfid_str = gf_strdup(uuid_utoa(gfdb_db_record->pargfid));
                if (!pargfid_str) {
                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
                                0, LG_MSG_CREATE_FAILED, "Creating pargfid_str "
                                "string failed.");
                        goto out;
                }

                /* Special performance case:
                 * Updating wind time in unwind for delete. This is done here
                 * as in the wind path we will not know whether its the last
                 * link or not. For a last link there is not use to update any
                 * wind or unwind time!*/
                if (gfdb_db_record->do_record_times) {
                        /*Update the wind write times*/
                        modtime = &gfdb_db_record->gfdb_wind_change_time;
                        ret = gf_update_time (sql_conn, gfid_str, modtime,
                                gfdb_db_record->do_record_counters,
                                _gf_true,
                                isreadfop (gfdb_db_record->gfdb_fop_type),
                                gfdb_db_record->ignore_errors);
                        if (ret) {
                                gf_msg (GFDB_STR_SQLITE3,
                                        _gfdb_log_level (GF_LOG_ERROR,
                                                gfdb_db_record->ignore_errors),
                                        0, LG_MSG_UPDATE_FAILED,
                                        "Failed update wind time in DB");
                                goto out;
                        }
                }

                modtime = &gfdb_db_record->gfdb_unwind_change_time;

                ret = gf_sql_delete_link(sql_conn, gfid_str, pargfid_str,
                                        gfdb_db_record->file_name,
                                        gfdb_db_record->ignore_errors);
                if (ret) {
                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                                LG_MSG_DELETE_FAILED, "Failed deleting link");
                        goto out;
                }

                if (gfdb_db_record->do_record_times &&
                        gfdb_db_record->do_record_uwind_time) {
                        ret = gf_update_time (sql_conn, gfid_str, modtime,
                                gfdb_db_record->do_record_counters,
                                _gf_false,
                                isreadfop(gfdb_db_record->gfdb_fop_type),
                                gfdb_db_record->ignore_errors);
                        if (ret) {
                                gf_msg (GFDB_STR_SQLITE3,
                                        _gfdb_log_level (GF_LOG_ERROR,
                                                gfdb_db_record->ignore_errors),
                                        0, LG_MSG_UPDATE_FAILED,
                                        "Failed update unwind time in DB");
                                goto out;
                        }
                }
        } else {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
                        0, LG_MSG_INVALID_UPLINK, "Invalid unlink option");
                goto out;
        }
        ret = 0;
out:
        GF_FREE (gfid_str);
        GF_FREE (pargfid_str);
        return ret;
}

/******************************************************************************
 *
 *                      Find/Query helper functions
 *
 * ****************************************************************************/
int
gf_sql_query_function (sqlite3_stmt              *prep_stmt,
                      gf_query_callback_t       query_callback,
                      void                      *_query_cbk_args)
{
        int ret                                         = -1;
        gfdb_query_record_t *query_record               = NULL;
        char *text_column                               = NULL;
        sqlite3 *db_conn                                = NULL;
        uuid_t  prev_gfid                               = {0};
        uuid_t  curr_gfid                               = {0};
        uuid_t  pgfid                                   = {0};
        char *base_name                                 = NULL;
        gf_boolean_t is_first_record                    = _gf_true;
        gf_boolean_t is_query_empty                     = _gf_true;

        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, prep_stmt, out);
        GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, query_callback, out);

        db_conn = sqlite3_db_handle(prep_stmt);

        /*
         * Loop to access queried rows
         * Each db record will have 3 columns
         * GFID, PGFID, FILE_NAME
         *
         * For file with multiple hard links we will get multiple query rows
         * with the same GFID, but different PGID and FILE_NAME Combination
         * For Example if a file with
         *         GFID = 00000000-0000-0000-0000-000000000006
         * has 3 hardlinks file1, file2 and file3 in 3 different folder
         * with GFID's
         * 00000000-0000-0000-0000-0000EFC00001,
         * 00000000-0000-0000-0000-00000ABC0001 and
         * 00000000-0000-0000-0000-00000ABC00CD
         * Then there will be 3 records
         *         GFID         : 00000000-0000-0000-0000-000000000006
         *         PGFID        : 00000000-0000-0000-0000-0000EFC00001
         *         FILE_NAME    : file1
         *
         *         GFID         : 00000000-0000-0000-0000-000000000006
         *         PGFID        : 00000000-0000-0000-0000-00000ABC0001
         *         FILE_NAME    : file2
         *
         *         GFID         : 00000000-0000-0000-0000-000000000006
         *         PGFID        : 00000000-0000-0000-0000-00000ABC00CD
         *         FILE_NAME    : file3
         *
         * This is retrieved and added to a single query_record
         *
         * query_record->gfid = 00000000-0000-0000-0000-000000000006
         *                  ->link_info = {00000000-0000-0000-0000-0000EFC00001,
         *                                 "file1"}
         *                                  |
         *                                  V
         *             link_info = {00000000-0000-0000-0000-00000ABC0001,
         *                                 "file2"}
         *                                  |
         *                                  V
         *             link_info = {00000000-0000-0000-0000-00000ABC0001,
         *                                 "file3",
         *                                 list}
         *
         * This query record is sent to the registered query_callback()
         *
         * */
        while ((ret = sqlite3_step (prep_stmt)) == SQLITE_ROW) {

                if (sqlite3_column_count(prep_stmt) > 0) {

                        is_query_empty = _gf_false;

                        /*Retrieving GFID - column index is 0*/
                        text_column = (char *)sqlite3_column_text
                                                        (prep_stmt, 0);
                        if (!text_column) {
                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                                        LG_MSG_GET_ID_FAILED, "Failed to"
                                        "retrieve GFID");
                                goto out;
                        }
                        ret = gf_uuid_parse (text_column, curr_gfid);
                        if (ret) {
                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                                        LG_MSG_PARSE_FAILED, "Failed to parse "
                                        "GFID");
                                goto out;
                        }

                        /*
                         * if the previous record was not of the current gfid
                         * call the call_back function and send the
                         * query record, which will have all the link_info
                         * objects associated with this gfid
                         *
                         * */
                        if (gf_uuid_compare (curr_gfid, prev_gfid) != 0) {

                                /* If this is not the first record */
                                if (!is_first_record) {
                                        /*Call the call_back function provided*/
                                        ret = query_callback (query_record,
                                                        _query_cbk_args);
                                        if (ret) {
                                                gf_msg (GFDB_STR_SQLITE3,
                                                        GF_LOG_ERROR, 0,
                                                LG_MSG_QUERY_CALL_BACK_FAILED,
                                                        "Query call back "
                                                        "failed");
                                                goto out;
                                        }

                                }

                                /*Clear the query record*/
                                gfdb_query_record_free (query_record);
                                query_record = NULL;
                                query_record = gfdb_query_record_new ();
                                if (!query_record) {
                                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
                                                0, LG_MSG_CREATE_FAILED,
                                                "Failed to create "
                                                "query_record");
                                        goto out;
                                }

                                gf_uuid_copy(query_record->gfid,
                                                                curr_gfid);
                                gf_uuid_copy(prev_gfid, curr_gfid);

                        }

                        /* Get PGFID */
                        text_column = (char *)sqlite3_column_text
                                                        (prep_stmt, 1);
                        if (!text_column) {
                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                                        LG_MSG_GET_ID_FAILED, "Failed to"
                                        " retrieve GF_ID");
                                goto out;
                        }
                        ret = gf_uuid_parse (text_column, pgfid);
                        if (ret) {
                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                                        LG_MSG_PARSE_FAILED, "Failed to parse "
                                        "GF_ID");
                                goto out;
                        }

                        /* Get Base name */
                        text_column = (char *)sqlite3_column_text
                                                        (prep_stmt, 2);
                        if (!text_column) {
                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                                        LG_MSG_GET_ID_FAILED, "Failed to"
                                        " retrieve GF_ID");
                                goto out;
                        }
                        base_name = text_column;


                        /* Add link info to the list */
                        ret = gfdb_add_link_to_query_record (query_record,
                                                         pgfid, base_name);
                        if (ret) {
                                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                                        LG_MSG_GET_ID_FAILED, "Failed to"
                                        " add link info to query record");
                                goto out;
                        }

                        is_first_record = _gf_false;

                }

        }

        if (ret != SQLITE_DONE) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                        LG_MSG_GET_RECORD_FAILED, "Failed to retrieve records "
                        "from db : %s", sqlite3_errmsg (db_conn));
                ret = -1;
                goto out;
        }


        if (!is_query_empty) {
                /*
                 * Call the call_back function for the last record from the
                 * Database
                 * */
                ret = query_callback (query_record, _query_cbk_args);
                if (ret) {
                        gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
                                LG_MSG_QUERY_CALL_BACK_FAILED,
                                "Query call back failed");
                        goto out;
                }
        }

        ret = 0;
out:
        gfdb_query_record_free (query_record);
        query_record = NULL;
        return ret;
}



int
gf_sql_clear_counters (gf_sql_connection_t *sql_conn)
{
        int ret                 = -1;
        char *sql_strerror      = NULL;
        char *query_str         = NULL;

        CHECK_SQL_CONN (sql_conn, out);

        query_str = "UPDATE "
                    GF_FILE_TABLE
                    " SET " GF_COL_READ_FREQ_CNTR " = 0 , "
                    GF_COL_WRITE_FREQ_CNTR " = 0 ;";

        ret = sqlite3_exec (sql_conn->sqlite3_db_conn, query_str, NULL, NULL,
                                &sql_strerror);
        if (ret != SQLITE_OK) {
                gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_EXEC_FAILED,
                        "Failed to execute: %s : %s",
                        query_str, sql_strerror);
                sqlite3_free (sql_strerror);
                        ret = -1;
                        goto out;
        }

        ret = 0;
out:
        return ret;
}